From e93b6a296c952a8d1be7bdfdf3c669557570e01f Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Sun, 9 Aug 2026 15:56:42 -0700 Subject: [PATCH 01/13] Add LMCache multiprocess API contract Signed-off-by: Yue Sun --- api/v1alpha1/cachebackend_types.go | 236 ++- api/v1alpha1/cachebackend_types_test.go | 124 +- api/v1alpha1/zz_generated.deepcopy.go | 215 +++ .../inferencecache.io_cachebackends.yaml | 1407 ++++++++++++++++- .../lmcache-multiprocess-migration-roadmap.md | 1093 +++++++++++++ internal/webhook/pod/podinjector.go | 27 + internal/webhook/pod/podinjector_test.go | 23 + .../cachebackend_defaulter_envtest_test.go | 31 + .../cachebackend_integration_validation.go | 11 + .../cachebackend_lmcache_mp_validation.go | 312 ++++ ...cachebackend_lmcache_mp_validation_test.go | 291 ++++ .../cachebackend_storage_validation.go | 15 + .../cachebackend_storage_validation_test.go | 28 + .../v1alpha1/cachebackend_validator.go | 2 + pkg/adapters/backend/backend.go | 25 +- pkg/adapters/backend/backend_test.go | 36 + pkg/adapters/runtime/adapter.go | 54 + pkg/adapters/runtime/adapter_test.go | 52 + 18 files changed, 3963 insertions(+), 19 deletions(-) create mode 100644 docs/design/lmcache-multiprocess-migration-roadmap.md create mode 100644 internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go create mode 100644 internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index 6089a94a..02a015fa 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -55,6 +55,28 @@ const ( CacheBackendRemoteStorageOwnershipExternal CacheBackendRemoteStorageOwnership = "External" ) +// +kubebuilder:validation:Enum=PodLocal;NodeLocal + +// LMCacheTopology identifies where the LMCache multiprocess server runs +// relative to the selected inference-engine Pods. LMCache is MP-only in the +// canonical API, so the process model is not repeated as an extra API level. +type LMCacheTopology string + +const ( + LMCacheTopologyPodLocal LMCacheTopology = "PodLocal" + LMCacheTopologyNodeLocal LMCacheTopology = "NodeLocal" +) + +// +kubebuilder:validation:Enum=Multiprocess + +// LMCacheConnectorMode identifies the connector protocol reflected in status. +// Multiprocess is the only canonical LMCache data plane. +type LMCacheConnectorMode string + +const ( + LMCacheConnectorModeMultiprocess LMCacheConnectorMode = "Multiprocess" +) + // +kubebuilder:validation:Enum=Deployment;StatefulSet // CacheBackendDeploymentKind identifies the Kubernetes workload kind used for managed backends. @@ -175,36 +197,158 @@ type CacheBackendHostMemorySpec struct { Capacity *resource.Quantity `json:"capacity,omitempty"` } +// LMCachePodLocalServerSpec configures the CacheBackend-owned LMCache MP +// server injected into each selected engine Pod. +type LMCachePodLocalServerSpec struct { + // Image is the digest-pinned LMCache server image. CacheBackend owns this + // cache component but never changes the inference-engine image. + Image string `json:"image"` + + // Port is the loopback port used by the engine-side connector. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + + // L1Capacity is the server's host-memory cache capacity. Container memory + // requests and limits must leave positive headroom above this value. + // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" + L1Capacity resource.Quantity `json:"l1Capacity"` + + // MaxWorkers bounds the MP server worker pool for this engine Pod. + // +kubebuilder:validation:Minimum=1 + MaxWorkers int32 `json:"maxWorkers"` + + // Resources are applied to the injected MP server container. Admission + // requires positive CPU and memory requests plus a memory limit that leaves + // headroom above l1Capacity. + Resources corev1.ResourceRequirements `json:"resources"` +} + +// LMCachePodLocalSpec configures one MP server per selected engine Pod. +type LMCachePodLocalSpec struct { + // Server is the CacheBackend-owned MP server configuration. + Server *LMCachePodLocalServerSpec `json:"server"` +} + +// LMCacheNodeLocalServerSpec describes the future one-server-per-node MP +// topology. The shape is published now so the API does not need another +// topology redesign, but admission rejects NodeLocal until Phase 8. +type LMCacheNodeLocalServerSpec struct { + Image string `json:"image"` + + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + Port int32 `json:"port"` + + // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" + L1Capacity resource.Quantity `json:"l1Capacity"` + + // MaxGPUWorkers bounds workers serving GPU-backed engine clients. + // +kubebuilder:validation:Minimum=1 + MaxGPUWorkers int32 `json:"maxGPUWorkers"` + + // MaxCPUWorkers bounds host-side storage workers. + // +kubebuilder:validation:Minimum=1 + MaxCPUWorkers int32 `json:"maxCPUWorkers"` + + Resources corev1.ResourceRequirements `json:"resources"` +} + +// LMCacheNodeLocalSchedulingSpec configures placement of the future per-node +// MP server workload. +type LMCacheNodeLocalSchedulingSpec struct { + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + + // +optional + Affinity *corev1.Affinity `json:"affinity,omitempty"` +} + +// LMCacheNodeLocalSpec configures the future one-server-per-eligible-node +// topology. It is rejected at admission until its controller exists. +type LMCacheNodeLocalSpec struct { + Server *LMCacheNodeLocalServerSpec `json:"server"` + + // +optional + Scheduling *LMCacheNodeLocalSchedulingSpec `json:"scheduling,omitempty"` +} + // LMCacheEngineSpec configures the engine-side LMCache implementation. These // fields apply to the connector or node-local MP worker, not to a remote // storage provider. type LMCacheEngineSpec struct { + // Topology selects the canonical LMCache MP server placement. PodLocal is + // implemented first; NodeLocal is reserved and rejected until Phase 8. + // Omit this field only for a legacy in-process/flat-field object during the + // repository migration window. + // +optional + Topology LMCacheTopology `json:"topology,omitempty"` + + // PodLocal configures one MP server in each selected engine Pod. + // +optional + PodLocal *LMCachePodLocalSpec `json:"podLocal,omitempty"` + + // NodeLocal configures a future per-node MP server. Admission currently + // rejects this block so it can never be accepted as inert configuration. + // +optional + NodeLocal *LMCacheNodeLocalSpec `json:"nodeLocal,omitempty"` + // ChunkSizeTokens is the number of tokens in an LMCache chunk. // +optional // +kubebuilder:validation:Minimum=1 ChunkSizeTokens *int32 `json:"chunkSizeTokens,omitempty"` - // HostMemory configures LMCache's engine-local host-memory tier. + // HostMemory is a legacy flat-field input retained only while repository + // consumers migrate to podLocal.server.l1Capacity. // +optional HostMemory *CacheBackendHostMemorySpec `json:"hostMemory,omitempty"` - // WorkerImage overrides the node-local LMCache MP worker image when the - // selected runtime uses multiprocess mode. + // WorkerImage is a legacy flat-field input retained only while repository + // consumers migrate to podLocal.server.image. // +optional WorkerImage string `json:"workerImage,omitempty"` - // WorkerPort overrides the node-local LMCache MP worker port. + // WorkerPort is a legacy flat-field input retained only while repository + // consumers migrate to podLocal.server.port. // +optional // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=65535 WorkerPort *int32 `json:"workerPort,omitempty"` - // RemoteSerde selects LMCache's serializer for a remote binding. + // RemoteSerde is a legacy in-process input and is forbidden with MP. // +optional RemoteSerde string `json:"remoteSerde,omitempty"` } -// RedisRemoteStorageSpec configures a Redis remote-storage provider. +// RemoteStorageTLSSpec configures server-authenticated TLS for a remote L3. +// Secret references are namespace-local to the CacheBackend. +type RemoteStorageTLSSpec struct { + // CACertificate selects a PEM CA bundle used to verify the provider. + CACertificate corev1.SecretKeySelector `json:"caCertificate"` + + // ServerName overrides the DNS name verified in the provider certificate. + // When omitted, clients verify the endpoint hostname. + // +optional + ServerName string `json:"serverName,omitempty"` +} + +// RedisAuthenticationSpec configures Redis ACL/password authentication without +// placing credentials directly in the CacheBackend or engine arguments. +type RedisAuthenticationSpec struct { + // Username selects the optional Redis ACL username. + // +optional + Username *corev1.SecretKeySelector `json:"username,omitempty"` + + // Password selects the required Redis password/token. + Password corev1.SecretKeySelector `json:"password"` +} + +// RedisRemoteStorageSpec configures a Redis remote-storage provider. Image and +// Resources are managed-workload settings; Authentication, TLS, and Database +// describe the engine/server binding for either ownership mode. type RedisRemoteStorageSpec struct { // Image is used only when ownership is Managed. // +optional @@ -213,6 +357,19 @@ type RedisRemoteStorageSpec struct { // Resources are applied to the managed Redis container. // +optional Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + + // Authentication references namespace-local Redis credentials. + // +optional + Authentication *RedisAuthenticationSpec `json:"authentication,omitempty"` + + // TLS configures certificate verification for Redis over TLS. + // +optional + TLS *RemoteStorageTLSSpec `json:"tls,omitempty"` + + // Database selects the Redis logical database. + // +optional + // +kubebuilder:validation:Minimum=0 + Database *int32 `json:"database,omitempty"` } // LMCacheServerRemoteStorageSpec configures a standalone lmcache-server @@ -733,9 +890,74 @@ type CacheBackendPodSpecOverride struct { TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` } +// CacheBackendConnectorStatus reports the engine-to-cache connector separately +// from the optional remote L3 provider. PodLocal loopback and NodeLocal +// node-derived addresses deliberately do not appear as a generic endpoint. +type CacheBackendConnectorStatus struct { + // Mode is Multiprocess for the canonical LMCache data plane. + Mode LMCacheConnectorMode `json:"mode,omitempty"` + + // Topology is the effective MP server placement. + Topology LMCacheTopology `json:"topology,omitempty"` + + // MatchedEnginePods is the number of selected engine Pods observed. + // +kubebuilder:validation:Minimum=0 + MatchedEnginePods int32 `json:"matchedEnginePods,omitempty"` + + // ReadyEnginePods is the number of selected engine Pods whose connector is + // ready. + // +kubebuilder:validation:Minimum=0 + ReadyEnginePods int32 `json:"readyEnginePods,omitempty"` + + // DesiredServers is one per selected engine Pod for PodLocal and one per + // eligible node for NodeLocal. + // +kubebuilder:validation:Minimum=0 + DesiredServers int32 `json:"desiredServers,omitempty"` + + // ReadyServers is the number of healthy MP servers. + // +kubebuilder:validation:Minimum=0 + ReadyServers int32 `json:"readyServers,omitempty"` + + // CoveredEnginePods is the number of selected engine Pods with a healthy, + // reachable MP server. + // +kubebuilder:validation:Minimum=0 + CoveredEnginePods int32 `json:"coveredEnginePods,omitempty"` + + // UncoveredEnginePods is the selected engine Pods without a healthy, + // reachable MP server. + // +kubebuilder:validation:Minimum=0 + UncoveredEnginePods int32 `json:"uncoveredEnginePods,omitempty"` +} + +// CacheBackendRemoteStorageStatus reports the optional shared L3 independently +// from connector health. Endpoint is meaningful here because an L3 provider is +// globally addressable; MP connector endpoints are Pod/node local and omitted. +type CacheBackendRemoteStorageStatus struct { + Provider CacheBackendRemoteStorageProvider `json:"provider,omitempty"` + + // +optional + Endpoint string `json:"endpoint,omitempty"` + + // Ready is True, False, or Unknown once the controller has evaluated the + // provider. + // +optional + Ready metav1.ConditionStatus `json:"ready,omitempty"` +} + // CacheBackendStatus defines the observed state of a cache backend. type CacheBackendStatus struct { - // Endpoint is the observed endpoint clients should use for this backend. + // Connector reports the engine-side connector and MP server topology. + // +optional + Connector *CacheBackendConnectorStatus `json:"connector,omitempty"` + + // RemoteStorage reports the optional remote L3 independently from the + // connector. + // +optional + RemoteStorage *CacheBackendRemoteStorageStatus `json:"remoteStorage,omitempty"` + + // Endpoint is the legacy remote-provider endpoint projection. New MP-aware + // clients read status.remoteStorage.endpoint; retained until repository + // consumers migrate. // +optional Endpoint string `json:"endpoint,omitempty"` diff --git a/api/v1alpha1/cachebackend_types_test.go b/api/v1alpha1/cachebackend_types_test.go index 4cd989a3..90f773a4 100644 --- a/api/v1alpha1/cachebackend_types_test.go +++ b/api/v1alpha1/cachebackend_types_test.go @@ -52,7 +52,7 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { // indexEntries was removed in #57 (it duplicated status.indexParticipation.prefixCount); // health was removed in an earlier change; capacity is removed in this PR. // All three are guarded by requireNoProperty checks below. - for _, field := range []string{"endpoint", "matchedEnginePods", "engineSelectorMessage", "failOpen", "conditions", "firstKVEventObservedAt", "firstAvailableAt"} { + for _, field := range []string{"connector", "remoteStorage", "endpoint", "matchedEnginePods", "engineSelectorMessage", "failOpen", "conditions", "firstKVEventObservedAt", "firstAvailableAt"} { if !hasProperty(statusSchema, field) { t.Fatalf("status.%s is missing from CRD schema", field) } @@ -82,6 +82,22 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { "StatefulSet", }) lmCacheSchema := mustProperty(t, specSchema, "lmCache") + requireNoProperty(t, lmCacheSchema, "multiprocess") + requireEnum(t, mustProperty(t, lmCacheSchema, "topology"), []string{"PodLocal", "NodeLocal"}) + podLocalSchema := mustProperty(t, lmCacheSchema, "podLocal") + requireRequired(t, podLocalSchema, "server") + podLocalServerSchema := mustProperty(t, podLocalSchema, "server") + for _, field := range []string{"image", "port", "l1Capacity", "maxWorkers", "resources"} { + requireRequired(t, podLocalServerSchema, field) + } + requireMinimum(t, mustProperty(t, podLocalServerSchema, "port"), 1) + requireMaximum(t, mustProperty(t, podLocalServerSchema, "port"), 65535) + requireMinimum(t, mustProperty(t, podLocalServerSchema, "maxWorkers"), 1) + nodeLocalSchema := mustProperty(t, lmCacheSchema, "nodeLocal") + requireRequired(t, nodeLocalSchema, "server") + nodeLocalServerSchema := mustProperty(t, nodeLocalSchema, "server") + requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxGPUWorkers"), 1) + requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxCPUWorkers"), 1) requireMinimum(t, mustProperty(t, lmCacheSchema, "chunkSizeTokens"), 1) requireMinimum(t, mustProperty(t, lmCacheSchema, "workerPort"), 1) requireMaximum(t, mustProperty(t, lmCacheSchema, "workerPort"), 65535) @@ -95,6 +111,22 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { t.Fatalf("spec.remoteStorage.%s is missing from CRD schema", field) } } + redisSchema := mustProperty(t, remoteStorageSchema, "redis") + for _, field := range []string{"authentication", "tls", "database"} { + if !hasProperty(redisSchema, field) { + t.Fatalf("spec.remoteStorage.redis.%s is missing from CRD schema", field) + } + } + requireMinimum(t, mustProperty(t, redisSchema, "database"), 0) + + connectorStatusSchema := mustProperty(t, statusSchema, "connector") + requireEnum(t, mustProperty(t, connectorStatusSchema, "mode"), []string{"Multiprocess"}) + requireEnum(t, mustProperty(t, connectorStatusSchema, "topology"), []string{"PodLocal", "NodeLocal"}) + for _, field := range []string{"matchedEnginePods", "readyEnginePods", "desiredServers", "readyServers", "coveredEnginePods", "uncoveredEnginePods"} { + requireMinimum(t, mustProperty(t, connectorStatusSchema, field), 0) + } + remoteStatusSchema := mustProperty(t, statusSchema, "remoteStorage") + requireEnum(t, mustProperty(t, remoteStatusSchema, "provider"), []string{"Redis", "LMCacheServer", "Mooncake"}) observationSchema := mustProperty(t, specSchema, "observation") for _, field := range []string{"modelID", "firstEventTimeout"} { if !hasProperty(observationSchema, field) { @@ -199,6 +231,96 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireMaximum(t, mustProperty(t, autoscalingSchema, "targetCPUUtilizationPercent"), 100) } +func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { + database := int32(2) + l1 := resource.MustParse("32Gi") + backend := &CacheBackend{ + Spec: CacheBackendSpec{ + Runtime: CacheBackendRuntimeSGLang, + Type: CacheBackendTypeLMCache, + LMCache: &LMCacheEngineSpec{ + Topology: LMCacheTopologyPodLocal, + PodLocal: &LMCachePodLocalSpec{Server: &LMCachePodLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, + L1Capacity: l1, + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("33Gi")}, + }, + }}, + NodeLocal: &LMCacheNodeLocalSpec{Scheduling: &LMCacheNodeLocalSchedulingSpec{ + NodeSelector: map[string]string{"pool": "cache"}, + Tolerations: []corev1.Toleration{{Key: "cache"}}, + }}, + }, + RemoteStorage: &CacheBackendRemoteStorageSpec{ + Provider: CacheBackendRemoteStorageProviderRedis, + Ownership: CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + Redis: &RedisRemoteStorageSpec{ + Database: &database, + Authentication: &RedisAuthenticationSpec{ + Password: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "password"}, + }, + TLS: &RemoteStorageTLSSpec{ + CACertificate: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-ca"}, Key: "ca.crt"}, + }, + }, + }, + }, + Status: CacheBackendStatus{ + Connector: &CacheBackendConnectorStatus{ + Mode: LMCacheConnectorModeMultiprocess, + Topology: LMCacheTopologyPodLocal, + MatchedEnginePods: 2, + ReadyEnginePods: 1, + DesiredServers: 2, + ReadyServers: 1, + }, + RemoteStorage: &CacheBackendRemoteStorageStatus{ + Provider: CacheBackendRemoteStorageProviderRedis, + Endpoint: "redis.example:6379", + Ready: metav1.ConditionTrue, + }, + }, + } + + data, err := json.Marshal(backend) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var roundTripped CacheBackend + if err := json.Unmarshal(data, &roundTripped); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(backend, &roundTripped) { + t.Fatalf("JSON round trip changed object\nwant: %#v\n got: %#v", backend, &roundTripped) + } + + copied := backend.DeepCopy() + backend.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("64Gi") + backend.Spec.LMCache.NodeLocal.Scheduling.NodeSelector["pool"] = "general" + backend.Spec.LMCache.NodeLocal.Scheduling.Tolerations[0].Key = "general" + *backend.Spec.RemoteStorage.Redis.Database = 9 + backend.Spec.RemoteStorage.Redis.Authentication.Password.Name = "changed" + backend.Status.Connector.ReadyServers = 2 + backend.Status.RemoteStorage.Endpoint = "changed:6379" + + if got := copied.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("33Gi")) != 0 { + t.Fatalf("podLocal server resources alias original: %s", got.String()) + } + if copied.Spec.LMCache.NodeLocal.Scheduling.NodeSelector["pool"] != "cache" || copied.Spec.LMCache.NodeLocal.Scheduling.Tolerations[0].Key != "cache" { + t.Fatalf("nodeLocal scheduling was not deep-copied") + } + if *copied.Spec.RemoteStorage.Redis.Database != 2 || copied.Spec.RemoteStorage.Redis.Authentication.Password.Name != "redis-auth" { + t.Fatalf("Redis binding was not deep-copied") + } + if copied.Status.Connector.ReadyServers != 1 || copied.Status.RemoteStorage.Endpoint != "redis.example:6379" { + t.Fatalf("MP status was not deep-copied") + } +} + func TestCacheBackendCRDPrintColumns(t *testing.T) { version := loadCacheBackendCRDVersion(t, "v1alpha1") columns := mustPath[[]any](t, version, "additionalPrinterColumns") diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b7278cfc..a18cf5d9 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -68,6 +68,21 @@ func (in *CacheBackendAutoscalingSpec) DeepCopy() *CacheBackendAutoscalingSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheBackendConnectorStatus) DeepCopyInto(out *CacheBackendConnectorStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendConnectorStatus. +func (in *CacheBackendConnectorStatus) DeepCopy() *CacheBackendConnectorStatus { + if in == nil { + return nil + } + out := new(CacheBackendConnectorStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendEngineSelector) DeepCopyInto(out *CacheBackendEngineSelector) { *out = *in @@ -307,6 +322,21 @@ func (in *CacheBackendRemoteStorageSpec) DeepCopy() *CacheBackendRemoteStorageSp return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheBackendRemoteStorageStatus) DeepCopyInto(out *CacheBackendRemoteStorageStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendRemoteStorageStatus. +func (in *CacheBackendRemoteStorageStatus) DeepCopy() *CacheBackendRemoteStorageStatus { + if in == nil { + return nil + } + out := new(CacheBackendRemoteStorageStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendSpec) DeepCopyInto(out *CacheBackendSpec) { *out = *in @@ -370,6 +400,16 @@ func (in *CacheBackendSpec) DeepCopy() *CacheBackendSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendStatus) DeepCopyInto(out *CacheBackendStatus) { *out = *in + if in.Connector != nil { + in, out := &in.Connector, &out.Connector + *out = new(CacheBackendConnectorStatus) + **out = **in + } + if in.RemoteStorage != nil { + in, out := &in.RemoteStorage, &out.RemoteStorage + *out = new(CacheBackendRemoteStorageStatus) + **out = **in + } if in.MatchedEnginePods != nil { in, out := &in.MatchedEnginePods, &out.MatchedEnginePods *out = new(int32) @@ -864,6 +904,16 @@ func (in *EngineInjectionOverrides) DeepCopy() *EngineInjectionOverrides { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LMCacheEngineSpec) DeepCopyInto(out *LMCacheEngineSpec) { *out = *in + if in.PodLocal != nil { + in, out := &in.PodLocal, &out.PodLocal + *out = new(LMCachePodLocalSpec) + (*in).DeepCopyInto(*out) + } + if in.NodeLocal != nil { + in, out := &in.NodeLocal, &out.NodeLocal + *out = new(LMCacheNodeLocalSpec) + (*in).DeepCopyInto(*out) + } if in.ChunkSizeTokens != nil { in, out := &in.ChunkSizeTokens, &out.ChunkSizeTokens *out = new(int32) @@ -891,6 +941,119 @@ func (in *LMCacheEngineSpec) DeepCopy() *LMCacheEngineSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMCacheNodeLocalSchedulingSpec) DeepCopyInto(out *LMCacheNodeLocalSchedulingSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(v1.Affinity) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheNodeLocalSchedulingSpec. +func (in *LMCacheNodeLocalSchedulingSpec) DeepCopy() *LMCacheNodeLocalSchedulingSpec { + if in == nil { + return nil + } + out := new(LMCacheNodeLocalSchedulingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMCacheNodeLocalServerSpec) DeepCopyInto(out *LMCacheNodeLocalServerSpec) { + *out = *in + out.L1Capacity = in.L1Capacity.DeepCopy() + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheNodeLocalServerSpec. +func (in *LMCacheNodeLocalServerSpec) DeepCopy() *LMCacheNodeLocalServerSpec { + if in == nil { + return nil + } + out := new(LMCacheNodeLocalServerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMCacheNodeLocalSpec) DeepCopyInto(out *LMCacheNodeLocalSpec) { + *out = *in + if in.Server != nil { + in, out := &in.Server, &out.Server + *out = new(LMCacheNodeLocalServerSpec) + (*in).DeepCopyInto(*out) + } + if in.Scheduling != nil { + in, out := &in.Scheduling, &out.Scheduling + *out = new(LMCacheNodeLocalSchedulingSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheNodeLocalSpec. +func (in *LMCacheNodeLocalSpec) DeepCopy() *LMCacheNodeLocalSpec { + if in == nil { + return nil + } + out := new(LMCacheNodeLocalSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMCachePodLocalServerSpec) DeepCopyInto(out *LMCachePodLocalServerSpec) { + *out = *in + out.L1Capacity = in.L1Capacity.DeepCopy() + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCachePodLocalServerSpec. +func (in *LMCachePodLocalServerSpec) DeepCopy() *LMCachePodLocalServerSpec { + if in == nil { + return nil + } + out := new(LMCachePodLocalServerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LMCachePodLocalSpec) DeepCopyInto(out *LMCachePodLocalSpec) { + *out = *in + if in.Server != nil { + in, out := &in.Server, &out.Server + *out = new(LMCachePodLocalServerSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCachePodLocalSpec. +func (in *LMCachePodLocalSpec) DeepCopy() *LMCachePodLocalSpec { + if in == nil { + return nil + } + out := new(LMCachePodLocalSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LMCacheServerRemoteStorageSpec) DeepCopyInto(out *LMCacheServerRemoteStorageSpec) { *out = *in @@ -1261,6 +1424,27 @@ func (in *PromptTemplateStatus) DeepCopy() *PromptTemplateStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RedisAuthenticationSpec) DeepCopyInto(out *RedisAuthenticationSpec) { + *out = *in + if in.Username != nil { + in, out := &in.Username, &out.Username + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + in.Password.DeepCopyInto(&out.Password) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisAuthenticationSpec. +func (in *RedisAuthenticationSpec) DeepCopy() *RedisAuthenticationSpec { + if in == nil { + return nil + } + out := new(RedisAuthenticationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RedisRemoteStorageSpec) DeepCopyInto(out *RedisRemoteStorageSpec) { *out = *in @@ -1269,6 +1453,21 @@ func (in *RedisRemoteStorageSpec) DeepCopyInto(out *RedisRemoteStorageSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + if in.Authentication != nil { + in, out := &in.Authentication, &out.Authentication + *out = new(RedisAuthenticationSpec) + (*in).DeepCopyInto(*out) + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(RemoteStorageTLSSpec) + (*in).DeepCopyInto(*out) + } + if in.Database != nil { + in, out := &in.Database, &out.Database + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RedisRemoteStorageSpec. @@ -1281,6 +1480,22 @@ func (in *RedisRemoteStorageSpec) DeepCopy() *RedisRemoteStorageSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RemoteStorageTLSSpec) DeepCopyInto(out *RemoteStorageTLSSpec) { + *out = *in + in.CACertificate.DeepCopyInto(&out.CACertificate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RemoteStorageTLSSpec. +func (in *RemoteStorageTLSSpec) DeepCopy() *RemoteStorageTLSSpec { + if in == nil { + return nil + } + out := new(RemoteStorageTLSSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicaCacheStatus) DeepCopyInto(out *ReplicaCacheStatus) { *out = *in diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 49daf3d2..96473cde 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -551,8 +551,9 @@ spec: minimum: 1 type: integer hostMemory: - description: HostMemory configures LMCache's engine-local host-memory - tier. + description: |- + HostMemory is a legacy flat-field input retained only while repository + consumers migrate to podLocal.server.l1Capacity. properties: capacity: anyOf: @@ -566,18 +567,1227 @@ spec: - message: capacity must be greater than zero rule: quantity(string(self)).isGreaterThan(quantity('0')) type: object + nodeLocal: + description: |- + NodeLocal configures a future per-node MP server. Admission currently + rejects this block so it can never be accepted as inert configuration. + properties: + scheduling: + description: |- + LMCacheNodeLocalSchedulingSpec configures placement of the future per-node + MP server workload. + properties: + affinity: + description: Affinity is a group of affinity scheduling + rules. + properties: + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in + the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same node, + zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added per-node + to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is + a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + tolerations: + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + server: + description: |- + LMCacheNodeLocalServerSpec describes the future one-server-per-node MP + topology. The shape is published now so the API does not need another + topology redesign, but admission rejects NodeLocal until Phase 8. + properties: + image: + type: string + l1Capacity: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: l1Capacity must be greater than zero + rule: quantity(string(self)).isGreaterThan(quantity('0')) + maxCPUWorkers: + description: MaxCPUWorkers bounds host-side storage workers. + format: int32 + minimum: 1 + type: integer + maxGPUWorkers: + description: MaxGPUWorkers bounds workers serving GPU-backed + engine clients. + format: int32 + minimum: 1 + type: integer + port: + format: int32 + maximum: 65535 + minimum: 1 + type: integer + resources: + description: ResourceRequirements describes the compute + resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - image + - l1Capacity + - maxCPUWorkers + - maxGPUWorkers + - port + - resources + type: object + required: + - server + type: object + podLocal: + description: PodLocal configures one MP server in each selected + engine Pod. + properties: + server: + description: Server is the CacheBackend-owned MP server configuration. + properties: + image: + description: |- + Image is the digest-pinned LMCache server image. CacheBackend owns this + cache component but never changes the inference-engine image. + type: string + l1Capacity: + anyOf: + - type: integer + - type: string + description: |- + L1Capacity is the server's host-memory cache capacity. Container memory + requests and limits must leave positive headroom above this value. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + x-kubernetes-validations: + - message: l1Capacity must be greater than zero + rule: quantity(string(self)).isGreaterThan(quantity('0')) + maxWorkers: + description: MaxWorkers bounds the MP server worker pool + for this engine Pod. + format: int32 + minimum: 1 + type: integer + port: + description: Port is the loopback port used by the engine-side + connector. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + resources: + description: |- + Resources are applied to the injected MP server container. Admission + requires positive CPU and memory requests plus a memory limit that leaves + headroom above l1Capacity. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry + in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - image + - l1Capacity + - maxWorkers + - port + - resources + type: object + required: + - server + type: object remoteSerde: - description: RemoteSerde selects LMCache's serializer for a remote - binding. + description: RemoteSerde is a legacy in-process input and is forbidden + with MP. + type: string + topology: + description: |- + Topology selects the canonical LMCache MP server placement. PodLocal is + implemented first; NodeLocal is reserved and rejected until Phase 8. + Omit this field only for a legacy in-process/flat-field object during the + repository migration window. + enum: + - PodLocal + - NodeLocal type: string workerImage: description: |- - WorkerImage overrides the node-local LMCache MP worker image when the - selected runtime uses multiprocess mode. + WorkerImage is a legacy flat-field input retained only while repository + consumers migrate to podLocal.server.image. type: string workerPort: - description: WorkerPort overrides the node-local LMCache MP worker - port. + description: |- + WorkerPort is a legacy flat-field input retained only while repository + consumers migrate to podLocal.server.port. format: int32 maximum: 65535 minimum: 1 @@ -777,6 +1987,66 @@ spec: redis: description: Redis contains Redis-owned configuration. properties: + authentication: + description: Authentication references namespace-local Redis + credentials. + properties: + password: + description: Password selects the required Redis password/token. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + description: Username selects the optional Redis ACL username. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + required: + - password + type: object + database: + description: Database selects the Redis logical database. + format: int32 + minimum: 0 + type: integer image: description: Image is used only when ownership is Managed. type: string @@ -839,6 +2109,43 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS configures certificate verification for Redis + over TLS. + properties: + caCertificate: + description: CACertificate selects a PEM CA bundle used + to verify the provider. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + description: |- + ServerName overrides the DNS name verified in the provider certificate. + When omitted, clients verify the endpoint hostname. + type: string + required: + - caCertificate + type: object type: object required: - ownership @@ -2370,9 +3677,67 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + connector: + description: Connector reports the engine-side connector and MP server + topology. + properties: + coveredEnginePods: + description: |- + CoveredEnginePods is the number of selected engine Pods with a healthy, + reachable MP server. + format: int32 + minimum: 0 + type: integer + desiredServers: + description: |- + DesiredServers is one per selected engine Pod for PodLocal and one per + eligible node for NodeLocal. + format: int32 + minimum: 0 + type: integer + matchedEnginePods: + description: MatchedEnginePods is the number of selected engine + Pods observed. + format: int32 + minimum: 0 + type: integer + mode: + description: Mode is Multiprocess for the canonical LMCache data + plane. + enum: + - Multiprocess + type: string + readyEnginePods: + description: |- + ReadyEnginePods is the number of selected engine Pods whose connector is + ready. + format: int32 + minimum: 0 + type: integer + readyServers: + description: ReadyServers is the number of healthy MP servers. + format: int32 + minimum: 0 + type: integer + topology: + description: Topology is the effective MP server placement. + enum: + - PodLocal + - NodeLocal + type: string + uncoveredEnginePods: + description: |- + UncoveredEnginePods is the selected engine Pods without a healthy, + reachable MP server. + format: int32 + minimum: 0 + type: integer + type: object endpoint: - description: Endpoint is the observed endpoint clients should use - for this backend. + description: |- + Endpoint is the legacy remote-provider endpoint projection. New MP-aware + clients read status.remoteStorage.endpoint; retained until repository + consumers migrate. type: string engineSelectorMessage: description: |- @@ -2544,6 +3909,28 @@ spec: transition rules (which changes do / do not cascade), and rate-limit / no-Ready / rollback / scale-up rationale. type: string + remoteStorage: + description: |- + RemoteStorage reports the optional remote L3 independently from the + connector. + properties: + endpoint: + type: string + provider: + description: |- + CacheBackendRemoteStorageProvider identifies the technology used for the + optional shared/remote cache tier. + enum: + - Redis + - LMCacheServer + - Mooncake + type: string + ready: + description: |- + Ready is True, False, or Unknown once the controller has evaluated the + provider. + type: string + type: object type: object type: object served: true diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md new file mode 100644 index 00000000..2d34434b --- /dev/null +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -0,0 +1,1093 @@ +# Design Roadmap: LMCache Multiprocess Migration + +Status: **engineering-validated; Phase 0 complete (2026-08-09)** · Scope: +deprecate and remove this project's LMCache in-process data plane, converge vLLM +and SGLang on LMCache multiprocess (MP) mode, and model +Pod-local and node-local MP server placement without conflating either with +optional remote storage. + +This roadmap is the tracking document for the migration. It supersedes the +long-term migration policy in +[`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md), while retaining that +document as implementation history and GPU-validation evidence for the current +SGLang prototype. It also drives the LMCache-related revisions to +[`cachebackend-api.md`](cachebackend-api.md). + +## Executive recommendation + +Do not delete the in-process (IP) implementation before a production-credible MP +replacement exists. Use a strangler migration: + +1. freeze the IP path and lock the target API; +2. introduce an MP-only `CacheBackend` surface; +3. extract and harden the common MP server infrastructure using SGLang; +4. add vLLM MP in explicit opt-in mode; +5. migrate all repository-owned samples, tests, and manifests; +6. confirm that the Phase 0 no-consumer assumption still holds, then remove the + IP adapter, `lm://` provider, and legacy lifecycle code; and +7. add node-local shared MP servers after the Pod-local path is stable. + +The final API does **not** preserve `InProcess` as a first-class long-term mode. +For `spec.type: LMCache`, the canonical data plane is MP. IP exists only as a +temporary implementation transition path, not an external compatibility +commitment under the Phase 0 finding. + +## Assumptions + +- `inferencecache.io/v1alpha1` remains pre-launch and eligible for the documented + alpha removal carve-out. The project owner confirmed on 2026-08-09 that there + are no external `CacheBackend` consumers and no installed legacy objects that + require migration. The selected policy is therefore an in-place alpha cleanup. + If that fact changes before physical removal, the conditional compatibility + stage in this roadmap becomes mandatory. +- The first complete MP milestone uses **PodLocal** placement. **NodeLocal** is a + designed topology but does not block removal of IP when cross-Pod sharing is + supplied by a supported remote L3. +- The initial remote L3 scope is RESP/Redis. MP + Mooncake Store, S3, NIXL, and + other adapters are separate follow-ups; they must not be implied by accepting + an inert provider declaration. +- Native sidecars require a supported Kubernetes version. The exact minimum + Kubernetes version is locked in Phase 0 and enforced/documented before the MP + path becomes the default. +- The inference-workload owner supplies and pins the engine image; CacheBackend + never rewrites it. CacheBackend pins the cache components it injects or + manages, and the selected runtime adapter validates the connector/server + compatibility profile. Mixed MP client/server versions are not assumed + wire-compatible. + +## Terminology and tier model + +LMCache upstream calls MP server CPU memory **L1** and secondary storage **L2**. +This project describes the complete serving hierarchy beginning with GPU KV, so +the corresponding project-level names are: + +| Project tier | Owner | LMCache upstream name | +|---|---|---| +| GPU KV / L1 | inference engine | engine GPU cache | +| Host-memory / L2 | IP connector or MP server | local CPU / MP L1 | +| Remote / L3 | optional storage adapter | remote backend / MP L2 | + +This roadmap uses the project-level names unless quoting an upstream flag or +type. In particular: + +- **MP server** is the out-of-process LMCache service reached by an engine + connector. Existing code and documents sometimes call it an MP worker. +- **PodLocal** means one MP server native sidecar per engine Pod, reached over + loopback and sharing that Pod's `/dev/shm`. +- **NodeLocal** means one MP server Pod per node, normally a DaemonSet member, + shared by engine Pods on that node. +- **Remote storage** means only the optional L3 behind the IP connector or MP + server. An MP server is never declared as `remoteStorage`. +- **Legacy LMCacheServer** means the legacy IP centralized-sharing service + reached through `remote_url: lm://...`. It is not an MP L3 adapter. +- **MP + Mooncake Store L3** means an MP server configured with + `--l2-adapter '{"type":"mooncake_store", ...}'`. It is distinct from the + legacy engine-side `mooncakestore://` connector. + +## Locked architectural decisions + +The following decisions are the baseline for implementation. Changing one +requires updating this roadmap and recording the replacement decision before +code lands. + +| ID | Decision | Rationale | +|---|---|---| +| D1 | The final LMCache data plane is MP-only. | Upstream v0.5.3 recommends MP but still documents IP. Deprecating IP is this project's decision; keeping both indefinitely doubles adapter, lifecycle, and test complexity. | +| D2 | `remoteStorage` is optional L3 only. | Local CPU capacity and MP server placement are engine-integration concerns, not remote-provider selection. | +| D3 | `LMCacheServer` is removed from the canonical `remoteStorage.provider` set. | `lm://` is a legacy IP remote connector and is absent from the MP L3 adapter catalog. | +| D4 | PodLocal is the first production candidate and migration target. | It has the smallest scheduling and ownership surface and builds on the existing SGLang proof. | +| D5 | NodeLocal means a per-`CacheBackend` DaemonSet in its first implementation. | Multiple engine Pods of one backend may share it; cross-`CacheBackend` sharing introduces unresolved config, tenancy, port, and deletion ownership. | +| D6 | A generic Deployment behind a load-balanced Service is not a valid CUDA MP topology. | CUDA IPC and shared memory require the engine to reach the MP server on its own node. | +| D7 | Connector endpoints are not published in the generic `status.endpoint`. | PodLocal uses loopback; NodeLocal is node-dependent. Only remote L3 has a globally meaningful provider endpoint. | +| D8 | Unsupported combinations are rejected at admission. | An accepted but inert cache field commonly produces silent zero-hit behavior. | +| D9 | Fail-open is rendered into runtime-native behavior and tested. | A custom environment variable without a known consumer is not an enforceable serving contract. | +| D10 | Provider restart recovery is capability-specific. | Legacy `lm://` socket recovery, MP server recovery, and remote L3 recovery have different semantics and blast radii. | +| D11 | Each supported vLLM profile explicitly identifies its MP connector implementation; the initial reference profile uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial profile uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future profile may validate a different implementation explicitly. | +| D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. Runtime adapters must declare and validate the connector capabilities they require without turning a tested engine image into an API allowlist or mutation default. | + +## Current state + +| Area | Current behavior | Gap to target | +|---|---|---| +| SGLang engine wire | Implicit MP; injects a Pod-local native sidecar, config file, loopback endpoint, and shared `/dev/shm`; the sidecar image defaults to the engine image. | Renderer is SGLang-private; cache-component ownership is coupled to the workload image; legacy ZMQ-only server entry point; incomplete worker health/recovery and parallelism coverage. | +| vLLM engine wire | `LMCacheConnectorV1` with optional host CPU, `lm://`, or `mooncakestore://`. | No `LMCacheMPConnector`; IP is still the only vLLM LMCache implementation. | +| CR API | MP mode is inferred from runtime. `hostMemory`, `workerImage`, `workerPort`, and `remoteSerde` are flat sibling fields. | No explicit MP topology; mode-specific fields can be accepted and ignored. | +| Remote storage | `Redis`, `LMCacheServer`, and `Mooncake` share one provider abstraction. | `LMCacheServer` is a legacy connector service, not a general MP L3; Mooncake needs a different MP binding shape. | +| Lifecycle | Every managed provider participates in the cache-server restart cascade. | Redis L3 restarts can roll engine fleets even though the engine connects to a local MP server. | +| Status | Provider readiness and engine-container crash loops are observed. | Native-sidecar health and node coverage are not represented; `status.endpoint` is ambiguous. | +| Tests | Strong Go unit coverage; SGLang single-GPU evidence; sample admission checks. | No default-install engine-Pod injection smoke; no vLLM MP; no automated GPU fault/parallelism matrix. | + +### Connector ownership + +The engine-side connector and the LMCache MP server are separate components, +and both engines require code from LMCache: + +| Runtime | Engine-owned integration surface | LMCache-owned dependency | +|---|---|---| +| vLLM | Generic `KVConnectorFactory` plus built-in `LMCacheConnectorV1` and `LMCacheMPConnector` registrations. An external module path can replace the registered implementation. | The built-in connectors still import the `lmcache` package; LMCache also ships its own vLLM connector implementation and the MP server. | +| SGLang | LMCache-specific `LMCRadixCache`, `--enable-lmcache`, and `--lmcache-config-file` integration. It is not selected through vLLM's generic connector registry. | SGLang imports `LMCacheMPConnector` and related adapters from `lmcache.integration.sglang`; LMCache also supplies the MP server. | + +Therefore neither engine image is self-sufficient merely because it exposes an +LMCache flag or connector class. A runtime adapter must verify that the engine +image contains the required LMCache client package/API, then CacheBackend injects +and manages a compatible MP server without replacing that engine image. + +Source support is not the same as image support. The upstream vLLM Dockerfile +defaults `INSTALL_KV_CONNECTORS=false`, so its connector source may be present +while the `lmcache` runtime dependency is absent. SGLang likewise documents +installing `lmcache` separately; its integration raises an error when that import +is unavailable. CacheBackend cannot fix a missing Python package by injecting +flags or a server sidecar. A supported runtime profile must therefore establish +that the workload image already contains the required LMCache client and expose +enough version/capability metadata for the adapter to select a compatible server. + +## Target architecture + +### PodLocal + +```text +engine Pod + +---------------------+ optional network +------------------+ + | vLLM or SGLang | | remote L3 | + | MP connector | | Redis initially | + | | | +---------^--------+ + | loopback + IPC | | + | v | | + | LMCache MP server +----------------------------------------+ + | CPU L2 / MP L1 | + +---------------------+ +``` + +Properties: + +- one MP server per engine Pod; +- engine endpoint is `127.0.0.1:`; +- L2 capacity is per engine Pod; +- engine and server share a Pod network namespace and `/dev/shm`; +- no `hostNetwork` is required; +- cross-Pod sharing requires a configured remote L3; +- lifecycle is coupled, but mid-flight server restart behavior must still be + defined and tested. + +### NodeLocal + +```text +GPU node + +-------------------+ +-------------------+ + | engine Pod A | | engine Pod B | + | MP connector | | MP connector | + +---------+---------+ +---------+---------+ + | node-local endpoint | + +-----------+-----------+ + v + +--------------------+ + | LMCache MP server | one DaemonSet Pod per node + | shared CPU L2 | + +---------+----------+ + | + v + optional remote L3 +``` + +Properties: + +- one MP server per eligible node per `CacheBackend`; +- multiple selected engine Pods on that node share CPU cache capacity; +- engines derive the endpoint from their node identity, not a load-balanced + Service endpoint; +- L2 capacity is per node; +- server scheduling, host port, host shared-memory arrangement, GPU visibility, + and node coverage become controller-owned concerns; +- cross-`CacheBackend` and cross-tenant sharing are out of scope for the first + implementation. + +## Target API direction + +The exact Go type names are finalized in Phase 1. The intended operator shape is: + +### PodLocal example + +```yaml +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-lmcache +spec: + runtime: VLLM + type: LMCache + lmCache: + topology: PodLocal + podLocal: + server: + image: registry.example/lmcache-vllm@sha256:... + port: 6555 + l1Capacity: 32Gi + maxWorkers: 1 + resources: + requests: + cpu: "2" + memory: 33Gi + limits: + memory: 33Gi + remoteStorage: + provider: Redis + ownership: External + endpoint: redis.example:6379 + integration: + role: ReadWrite + failOpen: true + engineSelector: + matchLabels: + app.kubernetes.io/name: vllm +``` + +Omit `remoteStorage` for host-only MP operation. + +### NodeLocal example + +```yaml +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-lmcache-node-local +spec: + runtime: VLLM + type: LMCache + lmCache: + topology: NodeLocal + nodeLocal: + server: + image: registry.example/lmcache-standalone@sha256:... + port: 6555 + l1Capacity: 128Gi + maxGPUWorkers: 8 + maxCPUWorkers: 8 + resources: + requests: + cpu: "8" + memory: 132Gi + limits: + memory: 132Gi + scheduling: + nodeSelector: + inferencecache.io/lmcache-mp: "true" + tolerations: [] + remoteStorage: + provider: Redis + ownership: External + endpoint: redis.example:6379 + engineSelector: + matchLabels: + app.kubernetes.io/name: vllm +``` + +### Final provider matrix + +| Engine | MP topology | No remote L3 | Redis/RESP | Mooncake Store | Legacy LMCacheServer | +|---|---|---:|---:|---:|---:| +| SGLang | PodLocal | required MVP | required MVP | future | rejected | +| vLLM | PodLocal | required MVP | required MVP | future | rejected | +| SGLang | NodeLocal | planned | planned | future | rejected | +| vLLM | NodeLocal | planned | planned | future | rejected | + +“Required MVP” means the combination must be implemented and validated, not that +the remote L3 field itself is required. + +## Status direction + +The final field names are part of Phase 1 API review. Status must distinguish +connector health from remote-provider health and must not compress a node-local +endpoint set into one string. + +```yaml +status: + connector: + mode: Multiprocess + topology: NodeLocal + matchedEnginePods: 8 + readyEnginePods: 8 + desiredServers: 4 + readyServers: 4 + coveredEnginePods: 8 + uncoveredEnginePods: 0 + remoteStorage: + provider: Redis + endpoint: redis.example:6379 + ready: "True" + conditions: + - type: ConnectorReady + status: "True" + - type: RemoteStorageReady + status: "True" + - type: Ready + status: "True" +``` + +Required semantics: + +- PodLocal `desiredServers` equals the selected engine Pod count. +- NodeLocal `desiredServers` equals the number of distinct nodes hosting selected + engine Pods, or the explicitly managed eligible-node count when the DaemonSet + is intentionally prewarmed. +- `coveredEnginePods` counts selected engine Pods whose required MP server is + healthy and reachable. +- PodLocal loopback and NodeLocal node-derived connector addresses are not + published as `status.remoteStorage.endpoint`. +- A configured but unavailable remote L3 produces an explicit degraded/remote + condition. Whether it changes overall `Ready` depends on the effective, + runtime-native fail-open policy. + +Condition and Event contract: + +| Signal | Semantics | +|---|---| +| `ConnectorReady` | `Unknown/ConnectorCapabilityUnverified` until the runtime declaration and Pod shape are verified; `False` for a known incompatibility or unhealthy required MP server; `True` only when the selected engines are covered by healthy MP servers. | +| `RemoteStorageReady` | Omitted when no L3 is configured; otherwise `Unknown/RemoteStoragePending`, `False/RemoteStorageUnavailable`, or `True/RemoteStorageReady`, independently of connector health. | +| `LegacyInProcessDeprecated` | Conditional compatibility signal only if a legacy consumer appears before physical removal: condition `True` plus a Warning Event with the same reason and a migration instruction. It is never set for typed MP objects. | + +Events are emitted on signal transitions or a changed observed generation, not +on every reconcile. The Phase 2 status writer implements the MP health signals; +the legacy deprecation writer is only implemented if Phase 6 is activated. + +## Delivery overview + +| Phase | Outcome | Depends on | Status | +|---|---|---|---| +| 0 | Design freeze, consumer audit, version/Kubernetes baseline | none | complete | +| 1 | MP-only API and admission/status contracts | Phase 0 | complete | +| 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | not started | +| 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | not started | +| 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | not started | +| 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | +| 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 finding | +| 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | not started | +| 8 | NodeLocal shared MP server topology | Phases 3–4; does not block Phase 7 | not started | + +## Phase 0 — design freeze and compatibility baseline + +### Goal + +Stop the target from moving while implementation begins and determine whether +the alpha removal carve-out is safe to use. + +### Deliverables + +- [x] Complete the engineering review and obtain project-owner approval for + D1–D12. +- [x] Inventory all repository manifests and confirm the external/installed + population of `CacheBackend` objects using: + - vLLM IP host-only; + - `remoteStorage.provider: LMCacheServer`; + - existing IP-only `remoteStorage.provider: Mooncake`; + - SGLang MP flat worker fields. +- [x] Decide the API migration strategy: + - **selected:** in-place `v1alpha1` cleanup because there are zero external + consumers and zero installed legacy objects; or + - served compatibility period if that fact changes before removal. +- [x] Pin and record the Phase 3/4 target tuple for each engine: + - reference engine image/digest, used only as a validation fixture; + - CacheBackend-injected server image/digest; + - LMCache version; + - CUDA/runtime version; + - Kubernetes version; + - target model and parallelism validation modes. +- [x] Freeze new feature work on `LMCacheConnectorV1`, `lm://`, and the managed + legacy LMCache server. +- [x] Resolve whether top-level managed-provider fields (`replicas`, + `autoscaling`, `deploymentKind`, `template`) move below `remoteStorage` in + the same alpha API cleanup. They must not accidentally configure an MP + server with a different lifecycle. + + **Selected:** retain them only as legacy provider-workload inputs during the + repository migration, then relocate any still-needed fields below a typed + `remoteStorage` managed-workload block in Phase 7. They never configure the + PodLocal/NodeLocal MP server; that lifecycle lives exclusively under + `lmCache.podLocal.server` or `lmCache.nodeLocal.server`. + +The locked Phase 3/4 validation targets are reference environments, not engine +image requirements or admission allowlists: + +| Component/profile | Reference validation environment | Ownership | Required validation | +|---|---|---|---| +| LMCache MP server | `lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13`; LMCache 0.5.3; linux/amd64; CUDA 13.0.1 | CacheBackend-injected and digest-pinned | Client/server compatibility with both reference runtimes, probes, restart, and recovery | +| vLLM connector profile | `lmcache/vllm-openai@sha256:dca0afdda6ad1bb02e63619d366fcd18975b334d7274739ed6f2025035865781`; LMCache 0.5.3; explicit `lmcache.integration.vllm.lmcache_mp_connector`; linux/amd64; CUDA 13.0.1 | Inference-owner image; test fixture only | Llama 3.1 8B; TP=1/2, plus TP=4 before common multi-GPU recommendation | +| SGLang connector profile | linux/amd64 from `lmsysorg/sglang@sha256:1c64fde976bdf0d56474a30bccbcfc19667e5b3ab34c826a534c9d6aaca41212` (`v0.5.13.post1-cu130`) with exactly `lmcache==0.5.3`; CUDA 13.0 | Inference-owner image; derived test fixture only | `--enable-lmcache`/config-file capability; Llama 3 8B; TP=1/2 | +| Redis L3 | `redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2` (`7.4.10-alpine`) | CacheBackend-managed when not external | Both engines: cross-Pod reuse, credentials/TLS, outage, and recovery | + +Kubernetes 1.29 with `SidecarContainers` enabled is the minimum; 1.33 or newer +is recommended. All four published registry digests resolved on 2026-08-09, and +the SGLang and Redis indexes contain linux/amd64 manifests. The SGLang derived +test-fixture digest is produced in Phase 3. The vLLM reference digest pins its +contents, but its upstream release build did not constrain the vLLM package +version; Phase 4 preflight must record `vllm.__version__` and reject that +reference profile if incompatible. None of these engine-image choices authorize +CacheBackend to mutate a workload's image. + +### Validation + +- [x] Repository search and sample inventory are attached to the design PR. +- [x] Current Go tests and sample verification pass before behavior changes. +- [x] Every unsupported or unverified combination has an explicit disposition. + +The repository manifest inventory is: + +| Migration class | Objects | Disposition | +|---|---:|---| +| vLLM IP + `LMCacheServer` | 13 | Migrate repository samples and references after vLLM MP passes Phase 4. | +| vLLM IP + existing engine-side Mooncake provider | 1 | Do not reinterpret as MP `mooncake_store`; migrate explicitly or defer until that adapter is supported. | +| vLLM IP host-only | 1 | Intentionally invalid test fixture; update with the admission tests. | +| SGLang MP with flat fields + Redis | 1 | Move to the typed PodLocal block in Phase 5. | +| SGLang MP with flat fields, host-only | 1 | Move to the typed PodLocal block in Phase 5. | +| EventsOnly, no LMCache data plane | 1 | No data-plane migration. | + +Direct repository consumers also include the reference-stack manifest and Helm +values, default-install checks, C2/C6 scripts, and their fixtures. The +`SGLangHiCache` sample is outside this LMCache migration. The project owner +confirmed that there are no external `CacheBackend` consumers and no installed +legacy objects requiring migration, so live-cluster inventory is not required. + +Baseline validation at commit `083e916` passed `go test ./...` and +`make verify-samples` (21 admitted, 2 explicit skips, 0 failures). Naming and +internal-reference checks also passed. These are API/data-plane baselines, not +GPU or kubelet-native-sidecar evidence; those gates remain in Phases 2–4. + +### Exit criteria + +- The project owner confirmed that no external or installed legacy consumer can + be broken by the chosen in-place alpha cleanup. +- The immutable candidate images, software versions, Kubernetes baseline, and + Phase 3/4 validation matrix are published. +- The target CR and migration policy are approved. + +## Phase 1 — MP-only API, admission, and status contract + +### Goal + +Add the final MP shape before changing the data plane, while retaining only the +minimum compatibility surface needed to migrate existing IP objects. + +### API work + +- [x] Add typed MP-only `spec.lmCache` configuration. Because MP is the only + canonical LMCache data plane, do not add a redundant `multiprocess` + nesting level. +- [x] Add the `PodLocal` topology and its typed server configuration. +- [x] Design the `NodeLocal` block now, but reject it until Phase 8 is + implemented. Do not accept inert NodeLocal objects. +- [x] Make host-only MP explicit by allowing `remoteStorage` to be absent. +- [x] Remove `LMCacheServer` from the canonical MP provider matrix while + retaining the legacy enum for topology-less repository objects until + Phase 7. +- [x] Define structured remote-provider bindings that can grow beyond + `Binding{Protocol, Endpoint}` to carry credentials, TLS, and adapter + parameters without stringly typed engine overrides. Typed Redis + credential/TLS/database fields are rejected until Phase 2 renders them, + so credentials cannot be accepted and silently ignored. +- [x] Define connector and remote-storage status separately. +- [x] Define how a workload declares or exposes its engine and LMCache-client + capability/version without allowing CacheBackend to rewrite the engine + image or making the admission webhook pull arbitrary registry content. +- [x] Define migration/deprecation conditions and Events. + +### Compatibility work + +- [x] Mark the old flat fields as legacy inputs: + - `lmCache.hostMemory`; + - `lmCache.workerImage`; + - `lmCache.workerPort`; + - `lmCache.remoteSerde`. +- [x] Prevent new objects from mixing old flat fields with the typed MP + topology. +- [x] Preserve the current runtime-derived behavior for existing objects until + Phase 6; do not silently switch existing vLLM IP objects to MP. +- [x] Define old-to-new field mappings in the migration table below. + +The runtime owner, not CacheBackend, selects and pins the inference image. Its +validated Pod template declares +`inferencecache.io/lmcache-connector-profile` and +`inferencecache.io/lmcache-client-version`. The image build pipeline must probe +the required connector import/entry point and record the package version before +publishing that declaration. At Pod admission, the selected typed MP adapter +compares the declaration with its required profile and validates observable +engine args/resources; admission never pulls the image or contacts a registry. +An absent/mismatched declaration, an adapter that has not implemented the typed +MP contract, or an unclassifiable engine topology is admitted fail-open without +cache mutation and with an actionable diagnostic. CacheBackend never rewrites +the engine image. The concrete profile probes and supported version tuples land +with the Phase 2 renderer and Phase 3/4 runtime adapters. + +| Legacy field | Typed MP disposition | +|---|---| +| `lmCache.hostMemory.capacity` | Copy to `lmCache.podLocal.server.l1Capacity`; separately choose explicit server resources with memory headroom. | +| `lmCache.workerImage` | Copy only after pinning it by digest to `lmCache.podLocal.server.image`. | +| `lmCache.workerPort` | Copy to `lmCache.podLocal.server.port` after collision validation. | +| `lmCache.remoteSerde` | No automatic mapping; remove it unless a future typed L3 adapter exposes and validates equivalent semantics. | +| `lmCache.chunkSizeTokens` | Remains `lmCache.chunkSizeTokens`; it is common connector configuration, not topology nesting. | +| `remoteStorage.provider: LMCacheServer` | No automatic provider mapping; explicitly select host-only or a supported L3 such as Redis. | + +`lmCache.podLocal.server.maxWorkers` and `resources` are new required choices; +legacy objects do not contain enough information to derive production-safe +values. + +### Admission invariants + +- [x] Exactly one topology-specific block matches `topology`. +- [x] `PodLocal` rejects `nodeLocal`; `NodeLocal` rejects `podLocal`. +- [x] MP rejects `remoteStorage.provider: LMCacheServer`. +- [x] SGLang and vLLM reject remote providers their selected MP adapter cannot + render. +- [x] L1 capacity is positive and has a schedulable memory budget with explicit + headroom. +- [x] Ports are valid and do not collide with known operator-owned MP/event + ports. +- [x] Version-sensitive or unvalidated parallelism combinations fail loudly at + the boundary where the topology is observable: CR admission for declared + fields, and engine-Pod admission for engine args/resources. A combination + that cannot be classified is not silently injected. +- [x] `remoteSerde` cannot be supplied to MP. +- [x] EventsOnly cannot carry MP or remote-storage configuration that will not + be used. + +### Tests + +- [x] CRD schema/defaulting unit tests. +- [x] Validating webhook table tests for every topology/provider combination. +- [x] Envtest CREATE/UPDATE compatibility tests. +- [x] Round-trip/deep-copy tests for all new typed fields. +- [x] Status serialization tests. Condition transitions land with the Phase 2 + status writer because Phase 1 intentionally changes no data plane. + +### Exit criteria + +- New PodLocal MP objects admit with or without Redis. +- Every impossible combination is rejected at admission. +- Existing IP objects still reconcile unchanged during the compatibility + period. +- Every MP field has an identified renderer/status consumer, and typed MP + objects cannot fall through to a legacy runtime adapter while those Phase 2-4 + consumers are landing. + +## Phase 2 — engine-neutral PodLocal MP server renderer + +### Goal + +Turn the existing SGLang-specific spike into shared infrastructure before vLLM +depends on it. + +### Refactoring work + +- [ ] Introduce an engine-neutral internal MP server configuration model. +- [ ] Extract native-sidecar, config-volume, `/dev/shm`, resources, probes, + security context, and L3 adapter rendering from + `sglang_lmcache_wire.go`. +- [ ] Keep engine launch surfaces separate: + - SGLang config file and `--enable-lmcache`; + - vLLM `LMCacheMPConnector` JSON and deterministic hash settings. +- [ ] Preserve atomic and idempotent Pod mutation. +- [ ] Preserve reserved-name and mount-collision checks. + +### Runtime work + +- [ ] Replace `python3 -m lmcache.v1.multiprocess.server` with the supported + `lmcache server` entry point for the pinned LMCache version. +- [ ] Add HTTP startup, readiness, and liveness probes. +- [ ] Expose/scrape Prometheus metrics. +- [ ] Add typed worker-pool sizing (`maxWorkers` initially; split GPU/CPU pools + when required by the pinned version and test matrix). +- [ ] Add explicit CPU, memory, and optional ephemeral-storage resources. +- [ ] Stop defaulting the MP sidecar to the engine image. Select the + CacheBackend-owned standalone server image by digest without modifying the + engine container image. +- [ ] Let each runtime adapter declare its required engine-side connector + capability and supported client/server profiles. Surface an explicit + warning/condition when the observed runtime cannot be verified. +- [ ] Render Redis credentials/TLS through structured binding before calling the + managed Redis path production-ready. + +### Lifecycle work + +- [ ] Add MP native-sidecar health observation from + `status.initContainerStatuses`. +- [ ] Stop treating every managed provider restart as an engine-restart event. +- [ ] Introduce capability-specific restart behavior for: + - MP server restart; + - Redis L3 restart; + - legacy `lm://` restart during the compatibility window. +- [ ] Define engine recovery behavior when the MP server restarts mid-flight. + +### Tests + +- [ ] Renderer unit tests independent of SGLang. +- [ ] Golden Pod tests for resources, probes, security, mounts, and L3 args. +- [ ] Re-injection/idempotence tests. +- [ ] Foreign volume/container collision tests. +- [ ] Kubernetes-version admission smoke for native-sidecar fields. +- [ ] Connector/remote-storage status condition-transition tests. + +### Exit criteria + +- SGLang uses the common renderer with no data-plane regression. +- The server exposes a real health endpoint and metrics. +- The controller can distinguish MP server failure from engine failure and + remote-L3 failure. +- No Redis restart causes an unconditional engine-fleet rollout. + +## Phase 3 — SGLang PodLocal MP production baseline + +### Goal + +Use the already working SGLang path to validate the common MP server under +parallelism and failure before adding vLLM. + +### Functional scope + +- [ ] SGLang + PodLocal + no remote L3. +- [ ] SGLang + PodLocal + managed Redis development profile. +- [ ] SGLang + PodLocal + external Redis production profile. +- [ ] ReadWrite role; reject unsupported role splits. +- [ ] Pinned SGLang/LMCache/CUDA image tuple. + +### Correctness work + +- [ ] Validate LMCache chunk size against the effective SGLang page size. +- [ ] Validate TP=1 and TP=2 at minimum. +- [ ] Prove store → engine-GPU flush → retrieve from MP L1. +- [ ] Prove cross-Pod store/retrieve through Redis with fresh engine and MP L1. +- [ ] Verify event hash-domain separation and routing behavior remain correct. +- [ ] Verify cache eviction cannot create an indefinitely silent stale-affinity + signal without an observable metric/condition. + +### Failure work + +- [ ] Kill the MP server process and verify the selected recovery policy. +- [ ] Hang the MP server and verify liveness recovery. +- [ ] Restart the Pod-local native sidecar without replacing the engine process. +- [ ] Stop, restart, and replace Redis. +- [ ] Exhaust or nearly exhaust MP L1 memory and verify bounded eviction rather + than node OOM. +- [ ] Verify fail-open behavior with runtime-native evidence. + +### Operability work + +- [ ] `ConnectorReady` reflects MP server health. +- [ ] `RemoteStorageReady` reflects Redis independently. +- [ ] Metrics prove lookup/store/retrieve/hit behavior. +- [ ] Logs identify engine Pod, backend, model, MP instance, and L3 adapter + without exposing credentials. +- [ ] Default-install smoke creates a matching SGLang engine Pod through the + live webhook and inspects the actual injected wire. + +### Exit criteria + +- All required SGLang GPU and failure tests pass on the pinned tuple. +- A restarted or unhealthy MP server cannot leave a Ready engine silently + caching nothing indefinitely. +- Redis loss degrades to the documented local behavior without unnecessary + engine rollout. +- The SGLang sample and design document match the implementation. + +## Phase 4 — vLLM PodLocal MP + +### Goal + +Provide the complete replacement for the current vLLM IP path before any IP +consumer is forced to migrate. + +### Engine wire + +- [ ] Add a dedicated vLLM MP adapter; do not mutate the legacy adapter in place. +- [ ] Render `LMCacheMPConnector` with: + - `kv_connector_module_path` selecting the pinned implementation required by + D11; + - `kv_role` derived from `integration.role`; + - `lmcache.mp.host=127.0.0.1`; + - the configured MP port; + - validated MQ timeout/heartbeat settings when exposed; + - runtime-native load-failure recompute/fail-open behavior. +- [ ] Preserve `PYTHONHASHSEED=0` across scheduler and worker processes. +- [ ] Reserve only correctness-critical args/env owned by the adapter. +- [ ] Reject hybrid/parallelism combinations not supported by the pinned + vLLM/LMCache tuple. + +### Functional scope + +- [ ] vLLM + PodLocal + no remote L3. +- [ ] vLLM + PodLocal + managed Redis development profile. +- [ ] vLLM + PodLocal + external Redis production profile. +- [ ] ReadOnly, WriteOnly, and ReadWrite roles. +- [ ] TP=1 and TP=2; TP=4 before recommending the topology for common multi-GPU + production workloads. +- [ ] Multi-server, DP + multi-server, and unsupported PP/MLA combinations are + rejected, not silently attempted. + +### Correctness and failure tests + +- [ ] Store → GPU flush → retrieve from Pod-local MP L1. +- [ ] Cross-Pod retrieve through Redis. +- [ ] TP hash determinism with a negative test showing the zero-hit failure when + `PYTHONHASHSEED` is not pinned. +- [ ] MP server crash/restart/re-registration. +- [ ] MP server hang/liveness recovery. +- [ ] Redis loss and recovery. +- [ ] Engine rollout while MP/L3 data remains available as designed. +- [ ] Version-skew negative test. + +### Exit criteria + +- vLLM MP passes the required GPU matrix below. +- The replacement provides a documented migration path for host-only IP and + centralized `lm://` users. +- MP becomes the recommended path in samples and operator docs. Legacy IP stays + implementation-only until repository migration and is then removed directly; + Phase 6 applies only if a legacy consumer appears before removal. + +## Phase 5 — migration tooling and consumer migration + +### Goal + +Make every legacy object's semantic change explicit. No automated migration may +silently remove cross-Pod cache sharing or select a different remote L3. + +### Conditional tooling + +Phase 0 found no external consumers or installed legacy objects, so migration +tooling is not a default deliverable. Build the following only if that fact +changes before removal: + +- [ ] Add a read-only inventory/doctor command that classifies every legacy + `CacheBackend` and prints its migration class. +- [ ] Add a dry-run manifest migration command or documented deterministic + transformation. +- [ ] Report fields that cannot be mapped automatically. +- [ ] Emit `LegacyInProcessDeprecated` status/Events for remaining legacy + objects. +- [ ] Provide rollback instructions during the compatibility window. + +### Migration classes + +| Existing object | Automatic portion | Required operator choice | +|---|---|---| +| SGLang MP with flat worker fields | Move image, port, and host-memory capacity into `lmCache.podLocal.server` and set `lmCache.topology: PodLocal`. | Confirm pinned image/resources and supported Kubernetes version. | +| vLLM IP host-only | Move host-memory capacity to PodLocal MP L1; select vLLM MP wire. | Confirm sidecar resources and accept the process/topology change. | +| vLLM IP + managed/external LMCacheServer | Preserve local capacity; remove `lm://`. | Select no L3 and lose cross-Pod sharing, or explicitly select a supported Redis/other L3. Never choose automatically. | +| vLLM IP + existing engine-side Mooncake provider | Preserve local intent only. | Wait for MP + Mooncake Store L3 support or migrate explicitly to Redis; URL config is not equivalent to MP adapter config. | +| Any IP object with `remoteSerde` | None. | Remove it or map it to a future typed L3 serde only when that adapter supports and validates the same semantics. | + +### Repository migration + +- [ ] Convert every canonical sample to MP. +- [ ] Convert reference-stack manifests. +- [ ] Replace IP documentation and screenshots. +- [ ] Replace IP unit/integration fixtures where they are not explicitly testing + the transition or a conditional Phase 6 compatibility window. +- [ ] Update support tables and CLI output. +- [ ] Remove language that calls the legacy LMCache server a CPU profile or the + default LMCache backend. + +### Exit criteria + +- Every repository-owned LMCache workload uses MP. +- If any external legacy object appears, it has an owner and migration + disposition. +- If migration tooling becomes necessary, it reports zero unknown/unclassified + legacy shapes. +- No migration silently changes cross-Pod sharing behavior. + +## Phase 6 — reject new IP objects + +**Conditional:** Phase 0 found no external consumers or installed legacy +objects. Skip this phase and proceed from Phase 5 to Phase 7 if that remains true. +Activate it in full if a legacy consumer or object appears before removal. + +### Goal + +Stop growth of the legacy population while allowing controlled migration or +deletion of existing objects. + +### Admission policy + +- [ ] Reject creation of vLLM LMCache objects without the MP block. +- [ ] Reject creation of `remoteStorage.provider: LMCacheServer`. +- [ ] Reject reintroduction of removed legacy fields. +- [ ] Grandfather existing IP objects only for: + - read/status; + - deletion; + - updates required to migrate to MP. +- [ ] Reject updates that scale out, materially retune, or otherwise extend the + lifetime/scope of a legacy IP deployment. + +### Operational gates + +- [ ] CLI/doctor reports remaining legacy object count. +- [ ] Release notes state the physical-removal target release. +- [ ] Warning Events link to migration documentation. +- [ ] A defined observation window passes with zero newly created IP objects. + +### Exit criteria + +- No supported API path can create a new IP data plane. +- Remaining legacy objects are zero, or each has an approved time-bounded + exception. +- MP error rate, hit behavior, and recovery behavior meet the agreed production + baseline. + +## Phase 7 — remove IP and the legacy LMCache server + +### Goal + +Delete the project-deprecated data plane and all code that exists solely to +operate it. + +### Code removal + +- [ ] Remove the vLLM legacy LMCache adapter. +- [ ] Remove `LMCacheConnectorV1` rendering. +- [ ] Remove `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, and other IP-only + injected settings. +- [ ] Remove `ProtocolLMCache` and the `lm://` endpoint parser/binding. +- [ ] Remove the managed and external `LMCacheServer` provider surface. +- [ ] Remove the standalone LMCache-server workload renderer. +- [ ] Remove the server-instance restart cascade if no remaining provider needs + it; otherwise narrow and rename it to the actual capability. +- [ ] Remove IP-only status fields, metrics, Events, samples, and tests. +- [ ] Remove compatibility defaulting/validation and migration-only code after + the supported migration window closes. + +### API cleanup + +- [ ] Remove legacy flat LMCache fields after their replacement is complete. +- [ ] Remove `LMCacheServer` from CRD enums and provider-specific schema. +- [ ] Remove or relocate top-level managed-provider workload fields according to + the Phase 0 decision. +- [ ] Regenerate CRDs, deepcopy code, examples, and reference documentation. + +### Verification + +- [ ] `go test ./...` passes. +- [ ] `make verify-samples` passes. +- [ ] Default-install and upgrade smoke pass. +- [ ] Repository search finds no production-code references to: + - `LMCacheConnectorV1`; + - `LMCACHE_REMOTE_URL`; + - `ProtocolLMCache`; + - `lm://`; + - the managed `LMCacheServer` provider. +- [ ] Migration documentation may retain historical references clearly marked as + removed. + +### Exit criteria + +- Only MP adapters can be selected for `spec.type: LMCache`. +- No controller workload or engine wire implements IP. +- No new or stored object requires the legacy schema to reconcile. + +## Phase 8 — NodeLocal shared MP servers + +### Goal + +Add the higher-efficiency topology in which multiple engine Pods of one +`CacheBackend` share a node-local MP server without weakening placement, +isolation, or status correctness. + +### Controller work + +- [ ] Reconcile one DaemonSet per NodeLocal `CacheBackend`. +- [ ] Restrict it to intended GPU/engine nodes through typed scheduling fields. +- [ ] Configure host networking and host shared memory according to the pinned + upstream deployment contract. +- [ ] Declare host ports so Kubernetes scheduling exposes conflicts. +- [ ] Authenticate ownership by CacheBackend name and UID. +- [ ] Compute desired/ready servers and engine-node coverage. +- [ ] Handle engine scheduling before the node-local server is ready without + starting an engine against a missing required MP endpoint. + +### Engine injection + +- [ ] Derive the node-local address from the engine Pod's node/host IP through a + Downward API field or another deterministic node-scoped mechanism. +- [ ] Do not use a load-balanced ClusterIP as the CUDA MP endpoint. +- [ ] Keep SGLang and vLLM launch surfaces engine-specific. +- [ ] Validate the server's global chunk size and version against every selected + engine Pod. + +### Isolation and resource work + +- [ ] Define port-conflict behavior for multiple NodeLocal CacheBackends on one + node. +- [ ] Restrict the first implementation to one trust/tenant domain per + CacheBackend server pool. +- [ ] Document that L1 capacity is per node and shared by selected engine Pods. +- [ ] Size `maxGPUWorkers` for the number of engine instances sharing a server. +- [ ] Add NetworkPolicy/firewall guidance where host networking permits it. +- [ ] Assess the security impact of exposing all node GPUs to the MP server. + +### Validation + +- [ ] One engine Pod on one node. +- [ ] Multiple engine Pods sharing one node-local server. +- [ ] Engines spread across multiple nodes, each using only its local server. +- [ ] DaemonSet rollout and single-node server restart. +- [ ] Node drain and engine rescheduling. +- [ ] Host-port conflict negative test. +- [ ] Redis outage/recovery with multiple node-local servers. +- [ ] No cross-node attempt to use CUDA IPC. + +### Exit criteria + +- Every selected engine Pod is covered by exactly one healthy local MP server. +- No generic Service load balancing can route an engine to another node's MP + server. +- Shared L1 behavior, resource accounting, and failure blast radius are measured + and documented. +- Cross-`CacheBackend` pool sharing remains rejected until a separate resource + and tenancy model is approved. + +## Required GPU validation matrix + +The matrix grows by phase. A cell is complete only when it proves a cache hit +after clearing or replacing the engine GPU cache; successful process startup is +not sufficient. + +| Runtime | Topology | Remote L3 | Parallelism | Required by | +|---|---|---|---|---| +| SGLang | PodLocal | none | TP=1 | Phase 3 | +| SGLang | PodLocal | Redis | TP=1 | Phase 3 | +| SGLang | PodLocal | Redis | TP=2 | Phase 3 | +| vLLM | PodLocal | none | TP=1 | Phase 4 | +| vLLM | PodLocal | Redis | TP=1 | Phase 4 | +| vLLM | PodLocal | Redis | TP=2 | Phase 4 | +| vLLM | PodLocal | Redis | TP=4 | Before production recommendation for common multi-GPU workloads | +| SGLang | NodeLocal | Redis | multiple engine Pods | Phase 8 | +| vLLM | NodeLocal | Redis | multiple engine Pods | Phase 8 | + +Every required data test records: + +- exact image digests and LMCache version; +- Kubernetes, driver, CUDA, and GPU model; +- engine args and effective MP server config; +- first-request store evidence; +- GPU-cache clear or fresh-engine proof; +- second-request retrieve/hit evidence; +- MP and L3 metrics before and after; +- failure/recovery timestamps where applicable. + +## Test pyramid + +| Layer | Required evidence | +|---|---| +| API/unit | schema, defaulting, validation, provider matrix, deep copy, status transitions | +| Renderer/unit | exact args/env/config, resources, probes, security, volumes, idempotence, collision rejection | +| Envtest | real CREATE/UPDATE admission and status persistence; legacy grandfathering only if Phase 6 activates | +| Kubernetes smoke | live webhook injection into matching engine Pods, native-sidecar schema support, controller-owned workload shape | +| GPU functional | store/flush/retrieve and cross-Pod L3 reuse | +| GPU fault | MP crash/hang/restart, Redis loss/recovery, engine rollout, node drain for NodeLocal | +| Upgrade/migration | Repository manifest conversion by default; old-object inventory, dry-run conversion, grandfather rules, and rollback only if Phase 6 activates | + +## Security, reliability, scalability, and cost gates + +### Security + +- [ ] No production managed Redis profile is exposed without an explicit network + isolation and credential/TLS posture. +- [ ] Secrets are referenced, not embedded in CR status, Pod args visible to all + readers, logs, or Events. +- [ ] PodLocal and NodeLocal GPU visibility is documented and reviewed for the + target tenancy model. +- [ ] NodeLocal host networking/IPC is an explicit operator choice. +- [ ] Cross-namespace remote endpoints retain explicit opt-in validation. + +### Reliability + +- [ ] MP server health affects connector status. +- [ ] A hung process is detected, not just an exited process. +- [ ] Recovery cannot leave the engine Ready with permanently disabled caching. +- [ ] Remote L3 loss follows tested fail-open/fail-closed behavior. +- [ ] Restart actions are scoped to the failing component's capability. + +### Scalability and latency + +- [ ] PodLocal memory cost is reported per engine Pod. +- [ ] NodeLocal memory cost is reported per node. +- [ ] Worker pool sizing is tested under the expected engine count and TP shape. +- [ ] Remote L3 concurrency and connection limits are bounded. +- [ ] Routing/index signals can be correlated with actual LMCache hit metrics. + +### Operability + +- [ ] Status distinguishes connector, MP server, engine, and remote L3 health. +- [ ] Metrics expose server availability, L1/L3 store/retrieve/hit, capacity, + eviction, and recovery. +- [ ] Events contain an actionable recovery or migration instruction. +- [ ] Samples never depend on an implicit runtime-selected connector mode. + +## Risk register + +| Risk | Impact | Mitigation / gate | +|---|---|---| +| A legacy consumer appears after Phase 0 | Breaking removal | Reconfirm before removal; activate Phase 6 and a grandfather period when non-zero. | +| MP client/server version skew | Permanent unhealthy or protocol failure | Same-image default for PodLocal, pinned matrix, skew-negative tests. | +| Worker restart does not re-register engine state | Ready engine silently misses forever | Runtime-specific recovery test and condition before Phase 3/4 exit. | +| Redis restart rolls all engines | Availability blast radius | Capability-specific restart policy in Phase 2. | +| `failOpen` is only a custom env | Contract not enforced | Render native runtime policy and fault-test it. | +| Sidecar sees all node GPUs | Isolation exposure | Document/review tenant model; prefer dedicated nodes where required. | +| PodLocal duplicates CPU L2 | Memory cost per replica | Explicit per-Pod capacity; NodeLocal follow-up. | +| NodeLocal port collision | DaemonSet Pods fail or bind incorrectly | Declared host port, typed port, controller condition, negative tests. | +| NodeLocal engine reaches remote node | CUDA IPC failure | Node-derived endpoint; reject load-balanced service topology. | +| Existing engine-side Mooncake config is treated as MP-equivalent | Admission succeeds but adapter cannot start | No automatic migration; separate MP + Mooncake Store implementation. | +| Index says warm while MP/L3 evicted data | Routing quality degrades silently | Correlate cache events with LMCache metrics/health; define stale-entry behavior. | + +## Phase tracking template + +Each implementation PR updates the delivery table and its phase checklist in the +same change. A phase is not marked complete merely because code merged. + +```markdown +### Phase N status update — YYYY-MM-DD + +- Status: not started | in progress | blocked | complete +- Owner: +- Tracking issue: +- PRs: +- Validation artifacts: +- Remaining exit criteria: +- Decision changes: +``` + +## Overall definition of done + +The migration is complete only when all of the following are true: + +- [ ] `spec.type: LMCache` selects only MP implementations. +- [ ] Both SGLang and vLLM pass the required PodLocal GPU matrix. +- [ ] Host-only and Redis-backed MP are supported for both engines. +- [ ] MP server health, failure, and recovery are observable and tested. +- [ ] `remoteStorage` is optional L3 and no longer contains LMCacheServer. +- [ ] No production code injects `LMCacheConnectorV1`, `lm://`, or + `LMCACHE_REMOTE_URL`. +- [ ] No generic managed-provider restart automatically rolls MP engines. +- [ ] Every old IP object has been migrated or intentionally deleted. +- [ ] Canonical samples, reference manifests, CLI output, and design documents + describe only the implemented MP behavior. +- [ ] NodeLocal, if enabled, guarantees same-node server selection and accurate + engine coverage; otherwise it remains rejected rather than partially + accepted. + +## Upstream references + +- [LMCache v0.5.3 release](https://github.com/LMCache/LMCache/releases/tag/v0.5.3) +- [LMCache MP overview](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/mp/index.rst) +- [LMCache MP deployment guide](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/mp/deployment.rst) +- [LMCache MP configuration](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/mp/configuration.rst) +- [LMCache MP supported storage adapters](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/mp/l2_storage/supported_storages.rst) +- [LMCache MP Mooncake Store adapter](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/mp/l2_storage/mooncake_store.rst) +- [Legacy IP centralized sharing](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/source/getting_started/quickstart/share_kv_cache.rst) +- [Official vLLM MP Kubernetes example](https://github.com/LMCache/LMCache/blob/v0.5.3/examples/multi_process/vllm-deployment.yaml) +- [MP worker liveness design](https://github.com/LMCache/LMCache/blob/v0.5.3/docs/design/v1/multiprocess/worker_liveness.md) +- [LMCache standalone image](https://github.com/LMCache/LMCache/blob/v0.5.3/docker/README.md#2-dockerfilestandalone---lmcache-only) +- [LMCache v0.5.3 vLLM MP connector](https://github.com/LMCache/LMCache/blob/v0.5.3/lmcache/integration/vllm/lmcache_mp_connector.py) +- [LMCache v0.5.3 SGLang MP adapter](https://github.com/LMCache/LMCache/blob/v0.5.3/lmcache/integration/sglang/multi_process_adapter.py) +- [vLLM v0.26.0 connector registry](https://github.com/vllm-project/vllm/blob/v0.26.0/vllm/distributed/kv_transfer/kv_connector/factory.py) +- [vLLM v0.26.0 built-in LMCache MP connector](https://github.com/vllm-project/vllm/blob/v0.26.0/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_mp_connector.py) +- [vLLM v0.26.0 Docker connector-dependency switch](https://github.com/vllm-project/vllm/blob/v0.26.0/docker/Dockerfile) +- [SGLang v0.5.13.post1 LMCache integration](https://github.com/sgl-project/sglang/blob/v0.5.13.post1/python/sglang/srt/mem_cache/storage/lmcache/README.md) +- [SGLang v0.5.13.post1 LMCache cache implementation](https://github.com/sgl-project/sglang/blob/v0.5.13.post1/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py) diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index 69f70dae..91d6c15b 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -173,6 +173,33 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi runtimeID, cache.Spec.Type, err)) } + // The typed LMCache topology is the Phase-1 boundary between the legacy + // in-process/flat-field wire and the final MP adapters. Never pass a typed MP + // object to a legacy adapter: doing so would silently inject the old vLLM IP + // connector or ignore the new PodLocal server settings. Until an adapter + // implements LMCacheMPRuntimeAdapter, admit the engine Pod untouched. + if cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" { + mpAdapter, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter) + if !ok { + log.V(1).Info("fail-open: selected adapter does not implement typed LMCache MP topology", + "runtime", string(runtimeID), "topology", string(cache.Spec.LMCache.Topology)) + return failOpen(req, &pod, fmt.Sprintf( + "runtime=%q adapter does not implement typed LMCache topology=%q (fail-open, no legacy injection)", + runtimeID, cache.Spec.LMCache.Topology)) + } + requirement := mpAdapter.ConnectorRequirement(cache) + if err := adapterruntime.ValidateConnectorDeclaration(&pod, requirement); err != nil { + log.V(1).Info("fail-open: engine connector capability is unverified", + "runtime", string(runtimeID), "error", err.Error()) + return failOpen(req, &pod, fmt.Sprintf("engine connector capability is unverified (fail-open): %v", err)) + } + if err := mpAdapter.ValidateMPEnginePod(&pod, cache); err != nil { + log.V(1).Info("fail-open: typed LMCache MP adapter rejected engine pod", + "runtime", string(runtimeID), "error", err.Error()) + return failOpen(req, &pod, fmt.Sprintf("typed LMCache MP compatibility check failed (fail-open): %v", err)) + } + } + endpoint := effectiveEndpoint(cache) storage := cache.Spec.EffectiveRemoteStorage() protocol, protocolErr := backendadapter.ProtocolFor(storage) diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index ada33ac3..cd49931d 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -319,6 +319,29 @@ func TestHandle_MatchAndInject(t *testing.T) { mustHaveArgFlag(t, mutated, "--kv-transfer-config") } +func TestHandle_TypedLMCacheDoesNotFallThroughToLegacyAdapter(t *testing.T) { + const ns = "engines" + cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) + cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + } + cb.Spec.RemoteStorage = nil + h := newHandler(t, cb) + pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) + req := newRequest(t, pod, ns) + + resp := h.Handle(context.Background(), req) + if !resp.Allowed { + t.Fatalf("typed MP pod must fail open while its adapter is pending: %+v", resp.Result) + } + if len(resp.Patches) != 0 { + t.Fatalf("typed MP pod must not receive legacy injection; got %d patches", len(resp.Patches)) + } + if resp.Result == nil || !strings.Contains(resp.Result.Message, "does not implement typed LMCache topology") { + t.Fatalf("response message = %v, want typed-topology adapter diagnostic", resp.Result) + } +} + func TestHandle_MatchAndInject_SGLang(t *testing.T) { // Covers the production pod-webhook selection path for (sglang, LMCache): // the nil-registry fallback now includes the SGLang adapter, so a SGLang diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go index 51802c53..95343b6e 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go @@ -246,6 +246,37 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) t.Errorf("operator replicas clobbered: got %v, want 5", explicit.Spec.Replicas) } + // --- Final MP API CREATE/UPDATE compatibility --- + // + // The real apiserver must accept the new PodLocal shape, preserve it on an + // unrelated update, and reject an update that mixes a legacy flat field into + // the canonical MP contract. + mpCR := validPodLocalMPBackend() + mpCR.Name = "podlocal-mp" + mpCR.Namespace = "team-a" + if err := k8s.Create(ctx, mpCR); err != nil { + t.Fatalf("PodLocal MP CacheBackend should be admitted: %v", err) + } + var persistedMP cachev1alpha1.CacheBackend + if err := live.Get(ctx, client.ObjectKey{Name: mpCR.Name, Namespace: mpCR.Namespace}, &persistedMP); err != nil { + t.Fatalf("get back PodLocal MP CR: %v", err) + } + if persistedMP.Spec.LMCache == nil || persistedMP.Spec.LMCache.Topology != cachev1alpha1.LMCacheTopologyPodLocal || + persistedMP.Spec.LMCache.PodLocal == nil || persistedMP.Spec.LMCache.PodLocal.Server == nil { + t.Fatalf("persisted PodLocal MP shape was lost: %+v", persistedMP.Spec.LMCache) + } + if persistedMP.Labels == nil { + persistedMP.Labels = map[string]string{} + } + persistedMP.Labels["phase"] = "one" + if err := k8s.Update(ctx, &persistedMP); err != nil { + t.Fatalf("unrelated update on PodLocal MP object should be admitted: %v", err) + } + persistedMP.Spec.LMCache.WorkerImage = "legacy-worker:test" + if err := k8s.Update(ctx, &persistedMP); err == nil { + t.Fatal("update mixing legacy workerImage into PodLocal MP should be rejected") + } + // --- Autoscaling defaulter-computed minReplicas --- // // Pins the one non-literal default: when an operator opts into diff --git a/internal/webhook/v1alpha1/cachebackend_integration_validation.go b/internal/webhook/v1alpha1/cachebackend_integration_validation.go index 02936658..ac3dee17 100644 --- a/internal/webhook/v1alpha1/cachebackend_integration_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_integration_validation.go @@ -391,6 +391,17 @@ func (v *CacheBackendValidator) checkRuntimeAdapter(cb *cachev1alpha1.CacheBacke ), } } + // A declared LMCache topology uses the Phase-1 MP support matrix validated + // by validateLMCacheTopology. The currently shipping runtime adapters still + // describe the legacy data plane (vLLM IP and the SGLang-specific MP spike), + // so consulting SupportsBinding here would incorrectly reject the final + // vLLM+Redis contract before the shared PodLocal renderer lands in Phases + // 2-4. Pod admission remains fail-open until the matching MP adapter is + // implemented; this exception is removed when both adapters expose the final + // binding capabilities. + if cb.Spec.LMCache != nil && cb.Spec.LMCache.Topology != "" { + return nil + } storage := cb.Spec.EffectiveRemoteStorage() protocol, err := backendadapter.ProtocolFor(storage) if err != nil { diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go new file mode 100644 index 00000000..8e531171 --- /dev/null +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -0,0 +1,312 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "fmt" + "regexp" + "strings" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation/field" +) + +const lmcacheKVEventPort int32 = 5557 + +var sha256ImagePattern = regexp.MustCompile(`^[^[:space:]@]+@sha256:[a-f0-9]{64}$`) + +// validateLMCacheTopology enforces the canonical MP-only LMCache shape while +// leaving a topology-less legacy object untouched during repository migration. +// The presence of topology/podLocal/nodeLocal is the explicit boundary between +// the old flat inputs and the new API; the two shapes can never be mixed. +func validateLMCacheTopology(cb *cachev1alpha1.CacheBackend) field.ErrorList { + if cb == nil || cb.Spec.LMCache == nil { + return nil + } + + lm := cb.Spec.LMCache + lmPath := field.NewPath("spec", "lmCache") + hasMPShape := lm.Topology != "" || lm.PodLocal != nil || lm.NodeLocal != nil + if !hasMPShape { + return nil // grandfathered flat-field/IP shape + } + + var errs field.ErrorList + if cb.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { + // validateCacheHierarchy owns the clearer type error. + return nil + } + if cb.Spec.IsEventsOnly() { + errs = append(errs, field.Forbidden(lmPath, + "LMCache topology is invalid with integration.mode=EventsOnly because that mode injects no KV connector or MP server")) + } + + // Flat fields are compatibility inputs, not alternate spellings for MP. + if lm.HostMemory != nil { + errs = append(errs, field.Forbidden(lmPath.Child("hostMemory"), + "legacy flat field cannot be mixed with the MP topology; use podLocal.server.l1Capacity")) + } + if strings.TrimSpace(lm.WorkerImage) != "" { + errs = append(errs, field.Forbidden(lmPath.Child("workerImage"), + "legacy flat field cannot be mixed with the MP topology; use podLocal.server.image")) + } + if lm.WorkerPort != nil { + errs = append(errs, field.Forbidden(lmPath.Child("workerPort"), + "legacy flat field cannot be mixed with the MP topology; use podLocal.server.port")) + } + if strings.TrimSpace(lm.RemoteSerde) != "" { + errs = append(errs, field.Forbidden(lmPath.Child("remoteSerde"), + "remoteSerde belongs to the legacy in-process connector and is not supported by LMCache MP")) + } + + storage := cb.Spec.EffectiveRemoteStorage() + if storage != nil { + switch storage.Provider { + case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: + // Redis/RESP is the initial shared L3 for both engines. + case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: + errs = append(errs, field.NotSupported( + field.NewPath("spec", "remoteStorage", "provider"), storage.Provider, + []string{string(cachev1alpha1.CacheBackendRemoteStorageProviderRedis)}, + )) + case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: + errs = append(errs, field.NotSupported( + field.NewPath("spec", "remoteStorage", "provider"), storage.Provider, + []string{string(cachev1alpha1.CacheBackendRemoteStorageProviderRedis)}, + )) + } + } + + switch lm.Topology { + case "": + errs = append(errs, field.Required(lmPath.Child("topology"), + "required when podLocal or nodeLocal is configured")) + case cachev1alpha1.LMCacheTopologyPodLocal: + if lm.PodLocal == nil { + errs = append(errs, field.Required(lmPath.Child("podLocal"), + "required when topology=PodLocal")) + } else if lm.PodLocal.Server == nil { + errs = append(errs, field.Required(lmPath.Child("podLocal", "server"), + "required when topology=PodLocal")) + } else { + errs = append(errs, validatePodLocalServer(lm.PodLocal.Server, lmPath.Child("podLocal", "server"))...) + } + if lm.NodeLocal != nil { + errs = append(errs, field.Forbidden(lmPath.Child("nodeLocal"), + "must be omitted when topology=PodLocal")) + } + case cachev1alpha1.LMCacheTopologyNodeLocal: + if lm.NodeLocal == nil { + errs = append(errs, field.Required(lmPath.Child("nodeLocal"), + "required when topology=NodeLocal")) + } else if lm.NodeLocal.Server == nil { + errs = append(errs, field.Required(lmPath.Child("nodeLocal", "server"), + "required when topology=NodeLocal")) + } else { + errs = append(errs, validateNodeLocalServer(lm.NodeLocal.Server, lmPath.Child("nodeLocal", "server"))...) + } + if lm.PodLocal != nil { + errs = append(errs, field.Forbidden(lmPath.Child("podLocal"), + "must be omitted when topology=NodeLocal")) + } + errs = append(errs, field.Forbidden(lmPath.Child("topology"), + "NodeLocal is reserved for Phase 8 and is not implemented; use PodLocal")) + default: + errs = append(errs, field.NotSupported(lmPath.Child("topology"), lm.Topology, + []string{string(cachev1alpha1.LMCacheTopologyPodLocal), string(cachev1alpha1.LMCacheTopologyNodeLocal)})) + } + + return errs +} + +func validatePodLocalServer(server *cachev1alpha1.LMCachePodLocalServerSpec, path *field.Path) field.ErrorList { + if server == nil { + return nil + } + errs := validateMPServer( + server.Image, + server.Port, + &server.L1Capacity, + server.Resources, + path, + ) + if server.MaxWorkers < 1 { + errs = append(errs, field.Invalid(path.Child("maxWorkers"), server.MaxWorkers, "must be at least 1")) + } + return errs +} + +func validateNodeLocalServer(server *cachev1alpha1.LMCacheNodeLocalServerSpec, path *field.Path) field.ErrorList { + if server == nil { + return nil + } + errs := validateMPServer( + server.Image, + server.Port, + &server.L1Capacity, + server.Resources, + path, + ) + if server.MaxGPUWorkers < 1 { + errs = append(errs, field.Invalid(path.Child("maxGPUWorkers"), server.MaxGPUWorkers, "must be at least 1")) + } + if server.MaxCPUWorkers < 1 { + errs = append(errs, field.Invalid(path.Child("maxCPUWorkers"), server.MaxCPUWorkers, "must be at least 1")) + } + return errs +} + +func validateMPServer( + image string, + port int32, + l1Capacity *resource.Quantity, + resources corev1.ResourceRequirements, + path *field.Path, +) field.ErrorList { + var errs field.ErrorList + trimmedImage := strings.TrimSpace(image) + switch { + case trimmedImage == "": + errs = append(errs, field.Required(path.Child("image"), "a CacheBackend-owned LMCache MP server image is required")) + case !sha256ImagePattern.MatchString(trimmedImage): + errs = append(errs, field.Invalid(path.Child("image"), image, + "must be pinned by sha256 digest (for example registry.example/lmcache@sha256:<64-hex-digest>)")) + } + + if port < 1 || port > 65535 { + errs = append(errs, field.Invalid(path.Child("port"), port, "must be between 1 and 65535")) + } else if port == lmcacheKVEventPort { + errs = append(errs, field.Invalid(path.Child("port"), port, + fmt.Sprintf("collides with the engine KV-event publisher port %d", lmcacheKVEventPort))) + } + + if l1Capacity == nil || l1Capacity.Sign() <= 0 { + var bad any + if l1Capacity != nil { + bad = l1Capacity.String() + } + errs = append(errs, field.Invalid(path.Child("l1Capacity"), bad, "must be greater than zero")) + } + errs = append(errs, validateMPServerResourceRequirements(resources, path.Child("resources"))...) + + cpuRequest, hasCPURequest := resources.Requests[corev1.ResourceCPU] + if !hasCPURequest || cpuRequest.Sign() <= 0 { + errs = append(errs, field.Required(path.Child("resources", "requests").Key(string(corev1.ResourceCPU)), + "a positive CPU request is required for the MP server")) + } + memoryRequest, hasMemoryRequest := resources.Requests[corev1.ResourceMemory] + if !hasMemoryRequest || memoryRequest.Sign() <= 0 { + errs = append(errs, field.Required(path.Child("resources", "requests").Key(string(corev1.ResourceMemory)), + "a positive memory request is required for the MP server")) + } else if l1Capacity != nil && l1Capacity.Sign() > 0 && memoryRequest.Cmp(*l1Capacity) <= 0 { + errs = append(errs, field.Invalid(path.Child("resources", "requests").Key(string(corev1.ResourceMemory)), + memoryRequest.String(), fmt.Sprintf("must be greater than l1Capacity %s so the server has explicit memory headroom", l1Capacity.String()))) + } + + memoryLimit, hasMemoryLimit := resources.Limits[corev1.ResourceMemory] + if !hasMemoryLimit || memoryLimit.Sign() <= 0 { + errs = append(errs, field.Required(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), + "a positive memory limit is required for the MP server")) + } else { + if hasMemoryRequest && memoryLimit.Cmp(memoryRequest) < 0 { + errs = append(errs, field.Invalid(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), + memoryLimit.String(), fmt.Sprintf("must be greater than or equal to the memory request %s", memoryRequest.String()))) + } + if l1Capacity != nil && l1Capacity.Sign() > 0 && memoryLimit.Cmp(*l1Capacity) <= 0 { + errs = append(errs, field.Invalid(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), + memoryLimit.String(), fmt.Sprintf("must be greater than l1Capacity %s so the server has explicit memory headroom", l1Capacity.String()))) + } + } + + return errs +} + +// validateMPServerResourceRequirements mirrors the generic provider-resource +// admission rules for the independently owned MP server resource block. Keep +// this at the API boundary: otherwise malformed extended resources are only +// discovered when the mutated engine Pod is submitted to the apiserver. +func validateMPServerResourceRequirements(resources corev1.ResourceRequirements, path *field.Path) field.ErrorList { + var errs field.ErrorList + if len(resources.Claims) > 0 { + errs = append(errs, field.Forbidden(path.Child("claims"), + "MP server resource claims are not supported because the injector does not own pod.spec.resourceClaims")) + } + + checkList := func(list corev1.ResourceList, kind string) { + for name, quantity := range list { + itemPath := path.Child(kind).Key(string(name)) + if msg, ok := validateContainerResourceName(name); !ok { + errs = append(errs, field.Invalid(itemPath, string(name), msg)) + continue + } + if quantity.Sign() < 0 { + errs = append(errs, field.Invalid(itemPath, quantity.String(), "must be a non-negative quantity")) + } + if !isOvercommittableResource(name) && !strings.HasPrefix(string(name), "hugepages-") { + if _, ok := quantity.AsInt64(); !ok { + errs = append(errs, field.Invalid(itemPath, quantity.String(), + fmt.Sprintf("%q is an extended resource and must be an integer quantity", name))) + } + } + if strings.HasPrefix(string(name), "hugepages-") && quantity.Sign() > 0 { + pageSize, err := resource.ParseQuantity(strings.TrimPrefix(string(name), "hugepages-")) + if err == nil && pageSize.Sign() > 0 && quantity.Value()%pageSize.Value() != 0 { + errs = append(errs, field.Invalid(itemPath, quantity.String(), + fmt.Sprintf("must be a multiple of the page size %s", pageSize.String()))) + } + } + } + } + checkList(resources.Requests, "requests") + checkList(resources.Limits, "limits") + + for name, request := range resources.Requests { + limit, hasLimit := resources.Limits[name] + if !isOvercommittableResource(name) && !hasLimit { + errs = append(errs, field.Invalid(path.Child("requests").Key(string(name)), request.String(), + fmt.Sprintf("%q is non-overcommittable and must also have an equal limit", name))) + continue + } + if !hasLimit { + continue + } + if isOvercommittableResource(name) && limit.Cmp(request) < 0 { + errs = append(errs, field.Invalid(path.Child("limits").Key(string(name)), limit.String(), + fmt.Sprintf("must be greater than or equal to request %s", request.String()))) + } + if !isOvercommittableResource(name) && limit.Cmp(request) != 0 { + errs = append(errs, field.Invalid(path.Child("limits").Key(string(name)), limit.String(), + fmt.Sprintf("must equal request %s for non-overcommittable resource %q", request.String(), name))) + } + } + return errs +} + +// rejectUnimplementedRedisBindingFeatures keeps the newly typed credential/TLS +// contract from becoming accepted-but-ignored configuration. Phase 2 removes +// this gate when the structured runtime binding and secret mounts are rendered. +func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) field.ErrorList { + if cb == nil || cb.Spec.RemoteStorage == nil || cb.Spec.RemoteStorage.Redis == nil { + return nil + } + redis := cb.Spec.RemoteStorage.Redis + path := field.NewPath("spec", "remoteStorage", "redis") + var errs field.ErrorList + if redis.Authentication != nil { + errs = append(errs, field.Forbidden(path.Child("authentication"), + "Redis authentication is typed but not rendered until Phase 2; refusing inert credentials")) + } + if redis.TLS != nil { + errs = append(errs, field.Forbidden(path.Child("tls"), + "Redis TLS is typed but not rendered until Phase 2; refusing inert TLS configuration")) + } + if redis.Database != nil { + errs = append(errs, field.Forbidden(path.Child("database"), + "Redis database selection is typed but not rendered until Phase 2; refusing inert adapter configuration")) + } + return errs +} diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go new file mode 100644 index 00000000..a9cfb5e6 --- /dev/null +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "context" + "strings" + "testing" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const testMPServerImage = "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func validPodLocalMPBackend() *cachev1alpha1.CacheBackend { + l1 := resource.MustParse("1Gi") + return &cachev1alpha1.CacheBackend{ + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{ + Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: testMPServerImage, + Port: 6555, + L1Capacity: l1, + MaxWorkers: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("3Gi"), + }, + }, + }, + }, + }, + }, + } +} + +func TestValidatorMPProviderMatrix(t *testing.T) { + tests := []struct { + name string + runtime cachev1alpha1.CacheBackendRuntime + provider cachev1alpha1.CacheBackendRemoteStorageProvider + wantErr bool + }{ + {name: "vLLM host-only", runtime: cachev1alpha1.CacheBackendRuntimeVLLM}, + {name: "SGLang host-only", runtime: cachev1alpha1.CacheBackendRuntimeSGLang}, + {name: "vLLM Redis", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis}, + {name: "SGLang Redis", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis}, + {name: "vLLM legacy LMCacheServer", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, wantErr: true}, + {name: "SGLang legacy LMCacheServer", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, wantErr: true}, + {name: "vLLM Mooncake future", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, wantErr: true}, + {name: "SGLang Mooncake future", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, wantErr: true}, + } + + validator := shippingValidator() + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Name = "mp" + cb.Namespace = "default" + cb.Spec.Runtime = tc.runtime + if tc.provider != "" { + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: tc.provider, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "storage.example:6379", + } + } + _, err := validator.ValidateCreate(context.Background(), cb) + if tc.wantErr && err == nil { + t.Fatal("expected admission rejection") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected admission rejection: %v", err) + } + }) + } +} + +func TestValidateLMCacheTopology(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend) + wantField string + }{ + {name: "PodLocal host-only"}, + { + name: "PodLocal Redis", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + } + }, + }, + { + name: "block without topology", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.Topology = "" + }, + wantField: "spec.lmCache.topology", + }, + { + name: "PodLocal missing block", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal = nil + }, + wantField: "spec.lmCache.podLocal", + }, + { + name: "PodLocal and NodeLocal mixed", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} + }, + wantField: "spec.lmCache.nodeLocal", + }, + { + name: "NodeLocal reserved", + mutate: func(cb *cachev1alpha1.CacheBackend) { + server := cb.Spec.LMCache.PodLocal.Server + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: server.Image, + Port: server.Port, + L1Capacity: server.L1Capacity, + MaxGPUWorkers: 1, + MaxCPUWorkers: 1, + Resources: server.Resources, + }, + } + }, + wantField: "spec.lmCache.topology", + }, + { + name: "legacy host memory mixed", + mutate: func(cb *cachev1alpha1.CacheBackend) { + capacity := resource.MustParse("1Gi") + cb.Spec.LMCache.HostMemory = &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &capacity} + }, + wantField: "spec.lmCache.hostMemory", + }, + { + name: "legacy worker image mixed", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.WorkerImage = "legacy:latest" + }, + wantField: "spec.lmCache.workerImage", + }, + { + name: "legacy serde mixed", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.RemoteSerde = "cachegen" + }, + wantField: "spec.lmCache.remoteSerde", + }, + { + name: "legacy LMCacheServer L3", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "cache.example:8200", + } + }, + wantField: "spec.remoteStorage.provider", + }, + { + name: "Mooncake L3 not implemented", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "mooncake.example:50051", + } + }, + wantField: "spec.remoteStorage.provider", + }, + { + name: "image tag is not immutable", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Image = "lmcache/standalone:v0.5.3" + }, + wantField: "spec.lmCache.podLocal.server.image", + }, + { + name: "digest without image name", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Image = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + wantField: "spec.lmCache.podLocal.server.image", + }, + { + name: "event port collision", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Port = lmcacheKVEventPort + }, + wantField: "spec.lmCache.podLocal.server.port", + }, + { + name: "memory request has no headroom", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Gi") + }, + wantField: "spec.lmCache.podLocal.server.resources.requests[memory]", + }, + { + name: "fractional extended resource", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") + }, + wantField: "spec.lmCache.podLocal.server.resources.requests[example.com/device]", + }, + { + name: "EventsOnly cannot carry MP", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly} + }, + wantField: "spec.lmCache", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := validPodLocalMPBackend() + if tc.mutate != nil { + tc.mutate(cb) + } + errs := validateLMCacheTopology(cb) + if tc.wantField == "" { + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + return + } + for _, err := range errs { + if err.Field == tc.wantField { + return + } + } + t.Fatalf("errors %v do not contain field %q", errs, tc.wantField) + }) + } +} + +func TestValidateLMCacheTopologyLeavesLegacyShapeUntouched(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.Topology = "" + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.WorkerImage = "legacy-worker:test" + if errs := validateLMCacheTopology(cb); len(errs) != 0 { + t.Fatalf("legacy topology-less shape should be left to compatibility rules: %v", errs) + } +} + +func TestRejectUnimplementedRedisBindingFeatures(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + Redis: &cachev1alpha1.RedisRemoteStorageSpec{ + Database: func() *int32 { v := int32(1); return &v }(), + }, + } + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 1 || !strings.Contains(errs[0].Field, "database") { + t.Fatalf("errors = %v, want database rejection", errs) + } + _, err := shippingValidator().ValidateCreate(context.Background(), cb) + if err == nil || !strings.Contains(err.Error(), "not rendered until Phase 2") { + t.Fatalf("ValidateCreate error = %v, want inert-binding rejection", err) + } + if strings.Contains(err.Error(), "provider workload configuration") { + t.Fatalf("external Redis connection settings were misclassified as managed workload config: %v", err) + } +} diff --git a/internal/webhook/v1alpha1/cachebackend_storage_validation.go b/internal/webhook/v1alpha1/cachebackend_storage_validation.go index 239098b6..f65fa91e 100644 --- a/internal/webhook/v1alpha1/cachebackend_storage_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_storage_validation.go @@ -114,6 +114,21 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { fmt.Sprintf("configuration belongs to provider %s, but remoteStorage.provider=%s", config.provider, storage.Provider))) } if config.set && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + if config.provider == cachev1alpha1.CacheBackendRemoteStorageProviderRedis { + // Redis combines connection settings (authentication/TLS/database), + // which apply to either ownership mode, with managed-workload + // settings. External bindings may retain the former but cannot ask + // this controller to choose an image or container resources. + if strings.TrimSpace(storage.Redis.Image) != "" { + errs = append(errs, field.Forbidden(config.path.Child("image"), + "valid only with Managed ownership")) + } + if storage.Redis.Resources != nil { + errs = append(errs, field.Forbidden(config.path.Child("resources"), + "valid only with Managed ownership")) + } + continue + } errs = append(errs, field.Forbidden(config.path, "provider workload configuration is valid only with Managed ownership")) } diff --git a/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go b/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go index c4ab7252..9f264be3 100644 --- a/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go @@ -146,6 +146,34 @@ func TestValidator_CanonicalCacheHierarchy(t *testing.T) { } }) + t.Run("external Redis separates binding from managed workload settings", func(t *testing.T) { + cb := newBackend() + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + Redis: &cachev1alpha1.RedisRemoteStorageSpec{ + Authentication: &cachev1alpha1.RedisAuthenticationSpec{ + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, + Key: "password", + }, + }, + }, + } + errs := validateCacheHierarchy(cb) + if len(errs) != 0 { + t.Fatalf("external Redis connection settings should be structurally valid: %v", errs) + } + + cb.Spec.RemoteStorage.Redis.Image = "redis:test" + errs = validateCacheHierarchy(cb) + if len(errs) != 1 || errs[0].Field != "spec.remoteStorage.redis.image" { + t.Fatalf("external Redis image errors = %v, want field-scoped managed-setting rejection", errs) + } + }) + t.Run("managed provider resources are validated at their typed path", func(t *testing.T) { cb := newBackend() cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang diff --git a/internal/webhook/v1alpha1/cachebackend_validator.go b/internal/webhook/v1alpha1/cachebackend_validator.go index 12916d78..4cca642a 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator.go +++ b/internal/webhook/v1alpha1/cachebackend_validator.go @@ -50,6 +50,8 @@ type ValidationRule func(cb *cachev1alpha1.CacheBackend) field.ErrorList // handler changes. var DefaultValidationRules = []ValidationRule{ validateCacheHierarchy, + validateLMCacheTopology, + rejectUnimplementedRedisBindingFeatures, rejectCrossNamespaceEndpointWithoutOptIn, requireExplicitMinReplicasOnScaleToZeroWithAutoscaling, rejectMooncakeMasterScaleOut, diff --git a/pkg/adapters/backend/backend.go b/pkg/adapters/backend/backend.go index 4e8c6878..61d3f923 100644 --- a/pkg/adapters/backend/backend.go +++ b/pkg/adapters/backend/backend.go @@ -30,6 +30,20 @@ const ( type Binding struct { Protocol Protocol Endpoint string + + // Redis carries typed RESP adapter configuration. Secret selectors remain + // references; credentials are never copied into the CacheBackend status or + // engine arguments. Nil for non-Redis bindings. + Redis *RedisBinding +} + +// RedisBinding is the runtime-facing, provider-specific portion of a RESP +// binding. It deliberately excludes managed-workload fields such as image and +// resources. +type RedisBinding struct { + Authentication *cachev1alpha1.RedisAuthenticationSpec + TLS *cachev1alpha1.RemoteStorageTLSSpec + Database *int32 } // RenderedStorage is the provider-owned workload shape. PodSpec and Service are @@ -87,7 +101,16 @@ func BindingFor(storage *cachev1alpha1.CacheBackendRemoteStorageSpec, protocol P if storage == nil { return nil } - return &Binding{Protocol: protocol, Endpoint: resolvedEndpoint} + binding := &Binding{Protocol: protocol, Endpoint: resolvedEndpoint} + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderRedis && storage.Redis != nil { + redis := storage.Redis.DeepCopy() + binding.Redis = &RedisBinding{ + Authentication: redis.Authentication, + TLS: redis.TLS, + Database: redis.Database, + } + } + return binding } // ProtocolFor returns the connection protocol associated with a provider. diff --git a/pkg/adapters/backend/backend_test.go b/pkg/adapters/backend/backend_test.go index 65000fd2..9e8c91b6 100644 --- a/pkg/adapters/backend/backend_test.go +++ b/pkg/adapters/backend/backend_test.go @@ -8,6 +8,7 @@ import ( "testing" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + corev1 "k8s.io/api/core/v1" ) func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { @@ -25,3 +26,38 @@ func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { t.Fatalf("binding endpoint = %q, want caller-resolved endpoint", got.Endpoint) } } + +func TestBindingForCarriesOnlyTypedRedisConnectionSettings(t *testing.T) { + database := int32(3) + storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + Redis: &cachev1alpha1.RedisRemoteStorageSpec{ + Image: "must-not-be-part-of-runtime-binding", + Database: &database, + Authentication: &cachev1alpha1.RedisAuthenticationSpec{ + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, + Key: "password", + }, + }, + }, + } + + got := BindingFor(storage, ProtocolRESP, "redis.example:6379") + if got == nil || got.Redis == nil { + t.Fatalf("BindingFor = %+v, want typed Redis binding", got) + } + if got.Redis.Authentication == nil || got.Redis.Authentication.Password.Name != "redis-auth" { + t.Fatalf("Redis authentication = %+v, want secret selector", got.Redis.Authentication) + } + if got.Redis.Database == nil || *got.Redis.Database != 3 { + t.Fatalf("Redis database = %v, want 3", got.Redis.Database) + } + *storage.Redis.Database = 9 + storage.Redis.Authentication.Password.Name = "changed" + if *got.Redis.Database != 3 || got.Redis.Authentication.Password.Name != "redis-auth" { + t.Fatalf("binding aliases mutable spec data: %+v", got.Redis) + } +} diff --git a/pkg/adapters/runtime/adapter.go b/pkg/adapters/runtime/adapter.go index feae9f1f..275784d7 100644 --- a/pkg/adapters/runtime/adapter.go +++ b/pkg/adapters/runtime/adapter.go @@ -111,6 +111,60 @@ type KVCacheRuntimeAdapter interface { EngineContainerName() string } +const ( + // AnnotationLMCacheConnectorProfile is the runtime-owner declaration of the + // engine-side LMCache connector API implemented by the workload image. + AnnotationLMCacheConnectorProfile = "inferencecache.io/lmcache-connector-profile" + // AnnotationLMCacheClientVersion declares the LMCache client package version + // validated by the runtime owner's image pipeline. + AnnotationLMCacheClientVersion = "inferencecache.io/lmcache-client-version" +) + +// LMCacheConnectorRequirement is the capability contract an MP runtime adapter +// requires from an inference workload. It names an interface profile rather +// than an image, so any inference system can provide a compatible image without +// CacheBackend owning or allowlisting that image. +type LMCacheConnectorRequirement struct { + Profile string + ClientVersion string +} + +// LMCacheMPRuntimeAdapter is the Phase-1 gate for adapters that understand the +// final typed LMCache topology. Legacy adapters intentionally do not implement +// it: the Pod webhook then admits a new MP Pod unmodified instead of silently +// applying the legacy in-process/flat-field wire. Phases 2-4 implement this +// interface as the shared renderer and runtime-specific MP adapters land. +type LMCacheMPRuntimeAdapter interface { + KVCacheRuntimeAdapter + + // ConnectorRequirement returns the workload-owned capability declaration + // required by this adapter. + ConnectorRequirement(*cachev1alpha1.CacheBackend) LMCacheConnectorRequirement + + // ValidateMPEnginePod validates version/parallelism/command/resource + // constraints visible only on the concrete engine Pod. An unclassifiable + // Pod returns an error and is never silently injected. + ValidateMPEnginePod(*corev1.Pod, *cachev1alpha1.CacheBackend) error +} + +// ValidateConnectorDeclaration compares the runtime owner's Pod annotations +// with an adapter's required connector contract. This is deliberately a +// declaration check, not registry/image introspection; build-time probes and +// digest pinning bind the claim to image contents outside the admission path. +func ValidateConnectorDeclaration(pod *corev1.Pod, requirement LMCacheConnectorRequirement) error { + if pod == nil { + return fmt.Errorf("engine pod is nil") + } + annotations := pod.GetAnnotations() + if got := annotations[AnnotationLMCacheConnectorProfile]; got != requirement.Profile { + return fmt.Errorf("annotation %s=%q, want %q", AnnotationLMCacheConnectorProfile, got, requirement.Profile) + } + if got := annotations[AnnotationLMCacheClientVersion]; got != requirement.ClientVersion { + return fmt.Errorf("annotation %s=%q, want %q", AnnotationLMCacheClientVersion, got, requirement.ClientVersion) + } + return nil +} + // ErrNoAdapter is returned by [Registry.Select] when no registered adapter // supports a given (runtime, CacheBackend) pair. An admission validator can // translate this into a user-visible rejection; the reconciler logs and skips. diff --git a/pkg/adapters/runtime/adapter_test.go b/pkg/adapters/runtime/adapter_test.go index 065e525b..0610cb16 100644 --- a/pkg/adapters/runtime/adapter_test.go +++ b/pkg/adapters/runtime/adapter_test.go @@ -344,6 +344,58 @@ func TestResolveRuntimeID(t *testing.T) { } } +func TestValidateConnectorDeclaration(t *testing.T) { + requirement := LMCacheConnectorRequirement{ + Profile: "sglang-lmcache-mp-v1", + ClientVersion: "0.5.3", + } + tests := []struct { + name string + annotations map[string]string + wantErr bool + }{ + { + name: "matching declaration", + annotations: map[string]string{ + AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", + AnnotationLMCacheClientVersion: "0.5.3", + }, + }, + {name: "missing declaration", wantErr: true}, + { + name: "profile mismatch", + annotations: map[string]string{ + AnnotationLMCacheConnectorProfile: "vllm-lmcache-mp-v1", + AnnotationLMCacheClientVersion: "0.5.3", + }, + wantErr: true, + }, + { + name: "version mismatch", + annotations: map[string]string{ + AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", + AnnotationLMCacheClientVersion: "0.5.2", + }, + wantErr: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: tc.annotations}} + err := ValidateConnectorDeclaration(pod, requirement) + if tc.wantErr && err == nil { + t.Fatal("expected declaration error") + } + if !tc.wantErr && err != nil { + t.Fatalf("unexpected declaration error: %v", err) + } + }) + } + if err := ValidateConnectorDeclaration(nil, requirement); err == nil { + t.Fatal("nil pod should be rejected") + } +} + func lookupEnv(env []corev1.EnvVar, name string) (string, bool) { for _, e := range env { if e.Name == name { From 089136c740d9e4367938fae6070f0046f6562824 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Sun, 9 Aug 2026 21:22:01 -0700 Subject: [PATCH 02/13] Add PodLocal LMCache multiprocess renderer Signed-off-by: Yue Sun --- README.md | 21 +- config/observability/kustomization.yaml | 15 +- config/observability/lmcache-podmonitor.yaml | 41 +++ config/observability/podmonitor.yaml | 4 +- .../lmcache-multiprocess-migration-roadmap.md | 82 +++-- docs/observability/alerts.md | 28 +- .../builtin/runtime/lmcache_mp_renderer.go | 346 ++++++++++++++++++ .../runtime/lmcache_mp_renderer_test.go | 259 +++++++++++++ .../builtin/runtime/sglang_lmcache.go | 106 +++++- .../builtin/runtime/sglang_lmcache_test.go | 97 +++++ .../builtin/runtime/sglang_lmcache_wire.go | 42 +-- internal/adapters/builtin/storage/redis.go | 47 ++- .../adapters/builtin/storage/redis_test.go | 56 +++ .../cachebackend_lmcache_mp_status.go | 236 ++++++++++++ .../cachebackend_lmcache_mp_status_test.go | 336 +++++++++++++++++ internal/controller/cachebackend_managed.go | 24 +- .../controller/cachebackend_reconciler.go | 3 + .../controller/cachebackend_serverless.go | 64 +++- internal/controller/cachebackend_status.go | 67 +++- internal/enginebinding/metadata.go | 16 + .../webhook/pod/envtest_integration_test.go | 81 +++- internal/webhook/pod/podinjector.go | 36 +- internal/webhook/pod/podinjector_test.go | 95 ++++- .../cachebackend_lmcache_mp_validation.go | 60 ++- ...cachebackend_lmcache_mp_validation_test.go | 100 ++++- site/content/en/docs/administration/_index.md | 2 +- .../observability-and-alerts.md | 12 +- site/content/en/docs/installation/_index.md | 8 +- site/content/en/docs/reference/metrics.md | 7 +- 29 files changed, 2148 insertions(+), 143 deletions(-) create mode 100644 config/observability/lmcache-podmonitor.yaml create mode 100644 internal/adapters/builtin/runtime/lmcache_mp_renderer.go create mode 100644 internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go create mode 100644 internal/controller/cachebackend_lmcache_mp_status.go create mode 100644 internal/controller/cachebackend_lmcache_mp_status_test.go diff --git a/README.md b/README.md index 994a335c..bd61e3e9 100644 --- a/README.md +++ b/README.md @@ -219,13 +219,13 @@ For prometheus-operator / kube-prometheus installs: kubectl apply -k config/observability ``` -This ships THREE resources: a `ServiceMonitor` (so Prometheus scrapes -`inference-cache-server:8080/metrics`), a `PodMonitor` (so Prometheus -scrapes the controller pod's `:8080/metrics` — required for the -controller-side alerts like `ServerProbeFail` to have a series to -evaluate), and the `PrometheusRule` carrying the alerts. +This ships FOUR resources: a `ServiceMonitor` for +`inference-cache-server:8080/metrics`, one `PodMonitor` for the controller +pod's `:8080/metrics`, one cross-namespace `PodMonitor` for successfully +injected PodLocal LMCache sidecars on their named `lmcache-http` port, and +the `PrometheusRule` carrying the alerts. -> **Caveat — Prometheus Operator selectors.** All three CRs carry +> **Caveat — Prometheus Operator selectors.** All four CRs carry > example labels (`prometheus: k8s`, plus `role: alert-rules` on the > PrometheusRule) that match the upstream kube-prometheus stack > (default `Prometheus` named `k8s`). The `kube-prometheus-stack` @@ -250,7 +250,7 @@ expressions) and only fire when the conditions are met. > **The fifth alert needs a vLLM scrape this bundle does NOT ship.** > [`LMCacheT2NoHits`](docs/observability/alerts.md#lmcachet2nohits) reads > `vllm:external_prefix_cache_*` from vLLM engine pods directly. The -> shipped `ServiceMonitor` covers only `inference-cache-server`. To make +> shipped scrape configs do not collect vLLM's own metrics. To make > that alert effective, add a separate `PodMonitor` for your vLLM > Deployment (or `kubernetes_sd_configs: pod` for vanilla Prometheus) > so engine `/metrics` is scraped with both `namespace` and `pod` labels @@ -258,11 +258,12 @@ expressions) and only fire when the conditions are met. For vanilla Prometheus, ConfigMap mounts, or Helm `prometheus.serverFiles`, use the flat [`alerting-rules.yaml`](config/observability/alerting-rules.yaml). -**You must also configure scraping yourself, for BOTH the server AND -the controller pod.** The server's `:8080` exposes the index, lookup, +**You must also configure scraping yourself for the server, the controller +pod, and every injected PodLocal LMCache sidecar.** The server's `:8080` exposes the index, lookup, and auth series; the controller pod's `:8080` exposes the per-stage probe-result counter (`inferencecache_backend_probe_result_total`) -and the cache-server restart-cascade counter — the controller-side +and the cache-server restart-cascade counter; each LMCache sidecar exposes +its own `lmcache_mp_*` series on `:8080/metrics` — the controller-side alerts (`ServerProbeFail` today) load against the controller's series, so a server-only scrape leaves them inert. diff --git a/config/observability/kustomization.yaml b/config/observability/kustomization.yaml index 16e5deec..94523595 100644 --- a/config/observability/kustomization.yaml +++ b/config/observability/kustomization.yaml @@ -2,9 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -# Optional observability overlay — applies BOTH scrape configs (a -# `ServiceMonitor` for the inference-cache-server and a `PodMonitor` -# for the inference-cache-controller) AND the alerting rules (a +# Optional observability overlay — applies all three scrape configs (a +# `ServiceMonitor` for the inference-cache-server, a `PodMonitor` +# for the inference-cache-controller, and a cross-namespace `PodMonitor` +# for injected LMCache MP native sidecars) AND the alerting rules (a # `PrometheusRule`) for the inference-cache alert bundle. Shipping # them together means a single `kubectl apply -k` wires the cache # plane into a prometheus-operator install end-to-end — Prometheus @@ -21,18 +22,19 @@ # kubectl apply -k config/observability # # For non-operator installs, see config/observability/alerting-rules.yaml -# (flat Prometheus rules file) plus your own scrape config covering BOTH +# (flat Prometheus rules file) plus your own scrape config covering # pods: `inference-cache-server.inference-cache-system.svc.cluster.local:8080` # (server-side series — index, lookup, auth) AND the # `inference-cache-controller-manager` pod's `:8080` (controller-side # series — per-stage probe-result counter, cache-server restart-cascade -# counter). The controller-side alerts (ServerProbeFail today) read +# counter) plus every injected PodLocal LMCache sidecar's `:8080/metrics`. +# The controller-side alerts (ServerProbeFail today) read # controller-emitted series, so a server-only scrape leaves them inert. # # Deliberately not aggregated into `config/default` so the default install # stays usable on clusters without prometheus-operator CRDs installed. # -# All three CRs are pinned to `inference-cache-system` (the operator +# All four CRs are pinned to `inference-cache-system` (the operator # namespace) so they land somewhere a typical prometheus-operator # install's `ruleNamespaceSelector` / `serviceMonitorNamespaceSelector` # / `podMonitorNamespaceSelector` already scans. If your Prometheus CR @@ -53,4 +55,5 @@ namespace: inference-cache-system resources: - servicemonitor.yaml - podmonitor.yaml + - lmcache-podmonitor.yaml - prometheus-rules.yaml diff --git a/config/observability/lmcache-podmonitor.yaml b/config/observability/lmcache-podmonitor.yaml new file mode 100644 index 00000000..e6afe524 --- /dev/null +++ b/config/observability/lmcache-podmonitor.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# PodMonitor for successfully injected PodLocal LMCache multiprocess native +# sidecars. The mutating webhook stamps the selector label only after the typed +# MP renderer has completed atomically, so matching Pods have the named +# `lmcache-http` port and the real FastAPI `/metrics` route. +# +# Engine Pods can live in any workload namespace while this PodMonitor remains +# in the inference-cache system/monitoring namespace. `namespaceSelector.any` +# is therefore intentional. Prometheus still needs RBAC to discover/scrape Pods +# in those namespaces, and a workload NetworkPolicy must allow traffic from the +# Prometheus namespace to TCP 8080. The endpoint is unauthenticated; do not +# expose it outside the cluster Pod network. +# +# Kubernetes Pod discovery includes declared init-container ports. That covers +# native sidecars, which are represented as initContainers with +# `restartPolicy: Always` but continue running for the Pod lifetime. +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: inference-cache-lmcache-mp + namespace: inference-cache-system + labels: + app.kubernetes.io/name: inference-cache + app.kubernetes.io/component: observability + # Must match the target Prometheus CR's podMonitorSelector. See the sibling + # controller PodMonitor for common kube-prometheus-stack alternatives. + prometheus: k8s +spec: + selector: + matchLabels: + inferencecache.io/lmcache-mp-metrics: "true" + namespaceSelector: + any: true + podMetricsEndpoints: + - port: lmcache-http + path: /metrics + interval: 30s + scrapeTimeout: 10s diff --git a/config/observability/podmonitor.yaml b/config/observability/podmonitor.yaml index 42d5eb12..f0814fa0 100644 --- a/config/observability/podmonitor.yaml +++ b/config/observability/podmonitor.yaml @@ -15,8 +15,8 @@ # controller binary does not front its /metrics endpoint with a Service — # the manager pod listens on :8080 directly. Loaded by the same # `kubectl apply -k config/observability` as the PrometheusRule and the -# server ServiceMonitor, so the alert bundle and BOTH scrape configs ship -# together. +# server ServiceMonitor and LMCache MP PodMonitor, so the alert bundle and all +# scrape configs ship together. # # Selector matches the controller Deployment in config/manager/manager.yaml. # The endpoint binds by port-NAME (`metrics`), not number, so this stays diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 2d34434b..069126ba 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -355,7 +355,7 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. |---|---|---|---| | 0 | Design freeze, consumer audit, version/Kubernetes baseline | none | complete | | 1 | MP-only API and admission/status contracts | Phase 0 | complete | -| 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | not started | +| 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | complete | | 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | not started | | 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | not started | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | @@ -481,8 +481,10 @@ minimum compatibility surface needed to migrate existing IP objects. - [x] Define structured remote-provider bindings that can grow beyond `Binding{Protocol, Endpoint}` to carry credentials, TLS, and adapter parameters without stringly typed engine overrides. Typed Redis - credential/TLS/database fields are rejected until Phase 2 renders them, - so credentials cannot be accepted and silently ignored. + credential/TLS/database fields initially reject unsupported use, so they + cannot be accepted and silently ignored; Phase 2 enables Secret-backed + SGLang RESP authentication while retaining explicit TLS/database + rejections for the pinned adapter. - [x] Define connector and remote-storage status separately. - [x] Define how a workload declares or exposes its engine and LMCache-client capability/version without allowing CacheBackend to rewrite the engine @@ -510,6 +512,10 @@ the required connector import/entry point and record the package version before publishing that declaration. At Pod admission, the selected typed MP adapter compares the declaration with its required profile and validates observable engine args/resources; admission never pulls the image or contacts a registry. +Successful mutation also stamps the CacheBackend generation rendered into the +immutable Pod. `ConnectorReady` treats an older generation as unverified until +the inference owner recreates or rolls that Pod; a CacheBackend spec update +cannot retroactively rewrite a running engine Pod. An absent/mismatched declaration, an adapter that has not implemented the typed MP contract, or an unclassifiable engine topology is admitted fail-open without cache mutation and with an actionable diagnostic. CacheBackend never rewrites @@ -576,53 +582,75 @@ depends on it. ### Refactoring work -- [ ] Introduce an engine-neutral internal MP server configuration model. -- [ ] Extract native-sidecar, config-volume, `/dev/shm`, resources, probes, +- [x] Introduce an engine-neutral internal MP server configuration model. +- [x] Extract native-sidecar, config-volume, `/dev/shm`, resources, probes, security context, and L3 adapter rendering from `sglang_lmcache_wire.go`. -- [ ] Keep engine launch surfaces separate: +- [x] Keep engine launch surfaces separate: - SGLang config file and `--enable-lmcache`; - vLLM `LMCacheMPConnector` JSON and deterministic hash settings. -- [ ] Preserve atomic and idempotent Pod mutation. -- [ ] Preserve reserved-name and mount-collision checks. +- [x] Preserve atomic and idempotent Pod mutation. +- [x] Preserve reserved-name and mount-collision checks. ### Runtime work -- [ ] Replace `python3 -m lmcache.v1.multiprocess.server` with the supported +- [x] Replace `python3 -m lmcache.v1.multiprocess.server` with the supported `lmcache server` entry point for the pinned LMCache version. -- [ ] Add HTTP startup, readiness, and liveness probes. -- [ ] Expose/scrape Prometheus metrics. -- [ ] Add typed worker-pool sizing (`maxWorkers` initially; split GPU/CPU pools +- [x] Add HTTP startup, readiness, and liveness probes. +- [x] Expose/scrape Prometheus metrics. +- [x] Add typed worker-pool sizing (`maxWorkers` initially; split GPU/CPU pools when required by the pinned version and test matrix). -- [ ] Add explicit CPU, memory, and optional ephemeral-storage resources. -- [ ] Stop defaulting the MP sidecar to the engine image. Select the +- [x] Add explicit CPU, memory, and optional ephemeral-storage resources. +- [x] Stop defaulting the MP sidecar to the engine image. Select the CacheBackend-owned standalone server image by digest without modifying the engine container image. -- [ ] Let each runtime adapter declare its required engine-side connector +- [x] Let each runtime adapter declare its required engine-side connector capability and supported client/server profiles. Surface an explicit warning/condition when the observed runtime cannot be verified. -- [ ] Render Redis credentials/TLS through structured binding before calling the - managed Redis path production-ready. +- [x] Render the Redis features supported by the pinned RESP adapter through + structured binding: Secret-backed authentication is wired on both ends; + unsupported TLS/database fields remain rejected instead of being silently + ignored. + +The exact LMCache 0.5.3 source constrains these runtime capability boundaries: + +- its `resp` adapter supports username/password (rendered from `SecretKeyRef`; + managed Redis supports the default user plus password), but it does not + support TLS or logical database selection. Admission therefore keeps + TLS/database rejected instead of accepting inert configuration. A future + validated Valkey adapter/image profile is required before those fields can be + used; +- `lmcache server` disables the separate Prometheus listener because its + FastAPI HTTP frontend already registers `/metrics` on `--http-port` (8080). + The renderer exposes the named `lmcache-http` port, successful typed PodLocal + injection stamps a stable metrics label, and the optional observability + overlay ships a cross-namespace `PodMonitor` for that label and route. ### Lifecycle work -- [ ] Add MP native-sidecar health observation from +- [x] Add MP native-sidecar health observation from `status.initContainerStatuses`. -- [ ] Stop treating every managed provider restart as an engine-restart event. -- [ ] Introduce capability-specific restart behavior for: +- [x] Stop treating every managed provider restart as an engine-restart event. +- [x] Introduce capability-specific restart behavior for: - MP server restart; - Redis L3 restart; - legacy `lm://` restart during the compatibility window. -- [ ] Define engine recovery behavior when the MP server restarts mid-flight. +- [x] Define the Phase 2 recovery boundary: report a native-sidecar outage + through `ConnectorReady` and rely on kubelet liveness restart. Phase 3 + GPU-validates whether the pinned SGLang connector re-registers without an + engine restart. ### Tests -- [ ] Renderer unit tests independent of SGLang. -- [ ] Golden Pod tests for resources, probes, security, mounts, and L3 args. -- [ ] Re-injection/idempotence tests. -- [ ] Foreign volume/container collision tests. -- [ ] Kubernetes-version admission smoke for native-sidecar fields. -- [ ] Connector/remote-storage status condition-transition tests. +- [x] Renderer unit tests independent of SGLang. +- [x] Golden Pod tests for resources, probes, security, mounts, and L3 args. +- [x] Re-injection/idempotence tests. +- [x] Foreign volume/container collision tests. +- [x] Kubernetes 1.31 envtest admission smoke for native-sidecar fields. +- [x] Connector/remote-storage status condition-transition tests. +- [x] Pinned LMCache 0.5.3 standalone-image smoke: the exact Phase 0 digest + starts `lmcache server` through its CPU fallback, `/healthcheck` returns + healthy, and `/metrics` returns Prometheus text on HTTP port 8080. ### Exit criteria diff --git a/docs/observability/alerts.md b/docs/observability/alerts.md index 80f852ba..4469dc8c 100644 --- a/docs/observability/alerts.md +++ b/docs/observability/alerts.md @@ -20,7 +20,7 @@ There are two distribution shapes, same rule set, drift-gated by kubectl apply -k config/observability ``` - Ships THREE CRs together — `kubectl apply -k` applies all three: + Ships FOUR CRs together — `kubectl apply -k` applies all four: 1. A [`ServiceMonitor`](../../config/observability/servicemonitor.yaml) that tells Prometheus to scrape `inference-cache-server:8080/metrics`. Without this, kube-prometheus installs will load the rules but @@ -35,10 +35,13 @@ There are two distribution shapes, same rule set, drift-gated by `inferencecache_backend_server_restart_cascades_total` is also controller-emitted). Without this, those rules load but never have a series to evaluate. - 3. The [`PrometheusRule`](../../config/observability/prometheus-rules.yaml) + 3. A second [`PodMonitor`](../../config/observability/lmcache-podmonitor.yaml) + that discovers successfully injected PodLocal LMCache native sidecars + across workload namespaces and scrapes their named `lmcache-http` port. + 4. The [`PrometheusRule`](../../config/observability/prometheus-rules.yaml) carrying the alerts. - All three CRs are pinned to namespace `inference-cache-system`. The + All four CRs are pinned to namespace `inference-cache-system`. The example selector labels each CR carries are: - `PrometheusRule` → `prometheus: k8s`, `role: alert-rules` (matched by @@ -46,11 +49,11 @@ There are two distribution shapes, same rule set, drift-gated by - `ServiceMonitor` → `prometheus: k8s` (matched by `Prometheus.spec.serviceMonitorSelector`). - - `PodMonitor` → + - both `PodMonitor` resources → `prometheus: k8s` (matched by `Prometheus.spec.podMonitorSelector`). - All three target the **upstream kube-prometheus stack**, whose default + All four target the **upstream kube-prometheus stack**, whose default `Prometheus` is named `k8s`. The `prometheus-community/kube-prometheus-stack` Helm chart uses a DIFFERENT convention — its selector matches `release: ` (no `prometheus:` label). Custom @@ -75,11 +78,12 @@ There are two distribution shapes, same rule set, drift-gated by [`alerting-rules.yaml`](../../config/observability/alerting-rules.yaml) into Prometheus via the `rule_files:` config block, a ConfigMap, or the Helm `prometheus.serverFiles` value (depending on your install). You ALSO - need `scrape_configs:` entries for BOTH targets — the + need `scrape_configs:` entries for all applicable targets — the `inference-cache-server` pod (server-side series: index, lookup, auth) AND the `inference-cache-controller-manager` pod (controller-side series: per-stage probe-result counter, cache-server restart-cascade - counter). Server-only scrape leaves the controller-side alerts + counter) and each injected PodLocal LMCache sidecar (`:8080/metrics`). + Server-only scrape leaves the controller-side alerts (`ServerProbeFail` today) loaded but inert — they read `inferencecache_backend_probe_result_total` which is controller-emitted. To keep the alerts' per-install scoping working, both scrapes must @@ -101,9 +105,9 @@ There are two distribution shapes, same rule set, drift-gated by scrapes one inference-cache install; do not use it for shared Prometheus deployments. - ServiceMonitor (server) + PodMonitor (controller) in the operator - bundle is the prometheus-operator equivalent of shape (1); both - shapes (1) and (2) require you to wire BOTH scrape entries + ServiceMonitor (server) + PodMonitors (controller and LMCache MP) in the operator + bundle are the prometheus-operator equivalent of shape (1); both + shapes (1) and (2) require you to wire all applicable scrape entries explicitly when you are not on prometheus-operator. Both files contain the same six active alerts (five Stage 1 alerts plus @@ -113,8 +117,8 @@ alerts](#deferred-alerts) below). > **One alert depends on a separate scrape this bundle does NOT ship.** > [`LMCacheT2NoHits`](#lmcachet2nohits) reads `vllm:external_prefix_cache_*`, -> which vLLM exposes on its own `/metrics`. The included `ServiceMonitor` -> covers only `inference-cache-server`. To make `LMCacheT2NoHits` light +> which vLLM exposes on its own `/metrics`. The included scrape configs do +> not collect vLLM's own metrics. To make `LMCacheT2NoHits` light > up, your install must also scrape engine pods — typically a separate > **`PodMonitor`** for your vLLM Deployment, or a `ServiceMonitor` on > a headless / per-pod Service (Endpoints discovery), or diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go new file mode 100644 index 00000000..27a3a60f --- /dev/null +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" + + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +// The PodLocal MP server wire is engine-neutral. Runtime adapters own only the +// engine launch surface that points at the returned client config path. +const ( + lmCacheMPServerContainerName = "lmcache-mp-server" + lmCacheMPServerManagedEnv = "INFERENCECACHE_MP_SERVER" + lmCacheMPServerManagedValue = "true" + + lmCacheMPConfigVolumeName = "lmcache-mp-config" + lmCacheMPConfigMountPath = "/var/run/inference-cache/lmcache" + lmCacheMPConfigFileName = "client.yaml" + lmCacheMPConfigFilePath = lmCacheMPConfigMountPath + "/" + lmCacheMPConfigFileName + + lmCacheMPShmVolumeName = "lmcache-mp-shm" + lmCacheMPShmMountPath = "/dev/shm" + + lmCacheMPServerPortName = "lmcache-mp" + lmCacheMPHTTPPortName = "lmcache-http" + lmCacheMPHTTPPort = int32(8080) + + lmCacheMPHealthPath = "/healthcheck" + + lmCacheRESPUsernameEnv = "LMCACHE_RESP_USERNAME" + lmCacheRESPPasswordEnv = "LMCACHE_RESP_PASSWORD" +) + +// lmCacheMPServerConfig is the internal, engine-neutral input to the PodLocal +// renderer. It is deliberately narrower than CacheBackend: the SGLang and vLLM +// adapters translate the public API into this model and keep their connector +// launch formats outside this file. +type lmCacheMPServerConfig struct { + Image string + Port int32 + ChunkSizeTokens int32 + L1Capacity resource.Quantity + MaxWorkers int32 + Resources corev1.ResourceRequirements + Binding *backendadapter.Binding + + // WriteClientConfig asks the native sidecar to create the generic LMCache + // MP client YAML shared with the engine. SGLang consumes this file. vLLM's + // connector consumes JSON and will set this false in Phase 4. + WriteClientConfig bool +} + +// renderLMCachePodLocalServer atomically injects the common PodLocal MP server +// wire and returns the generic client-config path when requested. A re-render +// converges resources, image, arguments, probes, and volumes in place. +func renderLMCachePodLocalServer(pod *corev1.PodSpec, engineContainerName string, cfg lmCacheMPServerConfig) (string, error) { + if pod == nil { + return "", fmt.Errorf("render LMCache MP server: pod spec is nil") + } + engineIndex, err := EngineContainerIndexNamed(pod, engineContainerName) + if err != nil { + return "", err + } + if err := validateLMCacheMPServerConfig(cfg); err != nil { + return "", err + } + + l2Adapter, bindingEnv, err := renderLMCacheMPL2Binding(cfg.Binding) + if err != nil { + return "", err + } + + // Mutate a copy and commit only after every collision/writability guard + // succeeds. Admission therefore receives either the complete wire or the + // original PodSpec, never a partially injected serving Pod. + work := pod.DeepCopy() + engine := &work.Containers[engineIndex] + owned := lmCacheMPWireIsOurs(pod) + + if legacy := findContainerByName(work.InitContainers, sglangMPWorkerContainerName); legacy != nil { + return "", fmt.Errorf("render LMCache MP server: pod already has legacy native sidecar %q; remove the legacy topology-less injection before enabling typed PodLocal", legacy.Name) + } + + if cfg.WriteClientConfig { + if existing := mountAtPath(engine.VolumeMounts, lmCacheMPConfigMountPath); existing != nil && + !(owned && existing.Name == lmCacheMPConfigVolumeName) { + return "", fmt.Errorf("render LMCache MP server: engine container already mounts %q from volume %q; that path is reserved for the LMCache MP client config", lmCacheMPConfigMountPath, existing.Name) + } + work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ + Name: lmCacheMPConfigVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, owned) + if err != nil { + return "", err + } + engine.VolumeMounts = upsertMountByName(engine.VolumeMounts, corev1.VolumeMount{ + Name: lmCacheMPConfigVolumeName, + MountPath: lmCacheMPConfigMountPath, + }) + } + + shmMount := corev1.VolumeMount{Name: lmCacheMPShmVolumeName, MountPath: lmCacheMPShmMountPath} + if existing := mountAtPath(engine.VolumeMounts, lmCacheMPShmMountPath); existing != nil && + !(owned && existing.Name == lmCacheMPShmVolumeName) { + if err := checkLMCacheMPShmReusable(work.Volumes, *existing); err != nil { + return "", err + } + shmMount = corev1.VolumeMount{ + Name: existing.Name, + MountPath: lmCacheMPShmMountPath, + SubPath: existing.SubPath, + } + } else { + l1 := cfg.L1Capacity.DeepCopy() + work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ + Name: lmCacheMPShmVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: &l1, + }}, + }, owned) + if err != nil { + return "", err + } + engine.VolumeMounts = upsertMountByName(engine.VolumeMounts, shmMount) + } + + server, err := lmCacheMPServerContainer(cfg, l2Adapter, bindingEnv, shmMount) + if err != nil { + return "", err + } + work.InitContainers, err = adoptContainer(work.InitContainers, server, owned) + if err != nil { + return "", err + } + + *pod = *work + if cfg.WriteClientConfig { + return lmCacheMPConfigFilePath, nil + } + return "", nil +} + +func validateLMCacheMPServerConfig(cfg lmCacheMPServerConfig) error { + switch { + case strings.TrimSpace(cfg.Image) == "": + return fmt.Errorf("render LMCache MP server: image is empty") + case cfg.Port < 1 || cfg.Port > 65535: + return fmt.Errorf("render LMCache MP server: port %d is outside 1-65535", cfg.Port) + case cfg.Port == lmCacheMPHTTPPort: + return fmt.Errorf("render LMCache MP server: MP port %d collides with HTTP health/control port", cfg.Port) + case cfg.ChunkSizeTokens < 1: + return fmt.Errorf("render LMCache MP server: chunkSizeTokens must be at least 1") + case cfg.L1Capacity.Sign() <= 0: + return fmt.Errorf("render LMCache MP server: l1Capacity must be greater than zero") + case cfg.MaxWorkers < 1: + return fmt.Errorf("render LMCache MP server: maxWorkers must be at least 1") + } + return nil +} + +func lmCacheMPServerContainer(cfg lmCacheMPServerConfig, l2Adapter string, bindingEnv []corev1.EnvVar, shmMount corev1.VolumeMount) (corev1.Container, error) { + l1GiB, err := quantityAsGiB(cfg.L1Capacity) + if err != nil { + return corev1.Container{}, err + } + + serverArgs := []string{ + "server", + "--host", "127.0.0.1", + "--port", strconv.FormatInt(int64(cfg.Port), 10), + "--http-host", "0.0.0.0", + "--http-port", strconv.FormatInt(int64(lmCacheMPHTTPPort), 10), + "--chunk-size", strconv.FormatInt(int64(cfg.ChunkSizeTokens), 10), + "--l1-size-gb", l1GiB, + "--eviction-policy", "LRU", + "--max-workers", strconv.FormatInt(int64(cfg.MaxWorkers), 10), + } + if l2Adapter != "" { + serverArgs = append(serverArgs, "--l2-adapter", l2Adapter) + } + + command := []string{"lmcache"} + args := serverArgs + mounts := []corev1.VolumeMount{shmMount} + if cfg.WriteClientConfig { + // Values are passed as positional arguments rather than interpolated into + // shell source. The shell only writes the SGLang-readable config and then + // execs the validated CLI argument vector unchanged. + const script = `set -eu +config_path="$1" +chunk_size="$2" +mp_port="$3" +shift 3 +printf 'chunk_size: %s\nmp_host: "127.0.0.1"\nmp_port: %s\n' "$chunk_size" "$mp_port" > "$config_path" +exec "$@"` + command = []string{"/bin/sh", "-c"} + args = []string{ + script, + "inference-cache-lmcache-config", + lmCacheMPConfigFilePath, + strconv.FormatInt(int64(cfg.ChunkSizeTokens), 10), + strconv.FormatInt(int64(cfg.Port), 10), + "lmcache", + } + args = append(args, serverArgs...) + mounts = append(mounts, corev1.VolumeMount{Name: lmCacheMPConfigVolumeName, MountPath: lmCacheMPConfigMountPath}) + } + + env := []corev1.EnvVar{ + {Name: "NVIDIA_VISIBLE_DEVICES", Value: "all"}, + {Name: lmCacheMPServerManagedEnv, Value: lmCacheMPServerManagedValue}, + } + for i := range bindingEnv { + env = UpsertEnv(env, bindingEnv[i]) + } + always := corev1.ContainerRestartPolicyAlways + return corev1.Container{ + Name: lmCacheMPServerContainerName, + Image: strings.TrimSpace(cfg.Image), + ImagePullPolicy: corev1.PullIfNotPresent, + RestartPolicy: &always, + Command: command, + Args: args, + Env: env, + Resources: *cfg.Resources.DeepCopy(), + VolumeMounts: mounts, + Ports: []corev1.ContainerPort{ + {Name: lmCacheMPServerPortName, ContainerPort: cfg.Port, Protocol: corev1.ProtocolTCP}, + {Name: lmCacheMPHTTPPortName, ContainerPort: lmCacheMPHTTPPort, Protocol: corev1.ProtocolTCP}, + }, + StartupProbe: lmCacheMPHTTPProbe(3, 40), + ReadinessProbe: lmCacheMPHTTPProbe(5, 3), + LivenessProbe: lmCacheMPHTTPProbe(10, 3), + SecurityContext: lmCacheMPServerSecurityContext(nil), + }, nil +} + +func lmCacheMPHTTPProbe(period, failures int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: lmCacheMPHealthPath, + Port: intstr.FromString(lmCacheMPHTTPPortName), + Scheme: corev1.URISchemeHTTP, + }}, + PeriodSeconds: period, + FailureThreshold: failures, + TimeoutSeconds: 2, + } +} + +func renderLMCacheMPL2Binding(binding *backendadapter.Binding) (string, []corev1.EnvVar, error) { + if binding == nil { + return "", nil, nil + } + if binding.Protocol != backendadapter.ProtocolRESP { + return "", nil, fmt.Errorf("render LMCache MP server: unsupported remote protocol %q", binding.Protocol) + } + if binding.Redis != nil { + if binding.Redis.TLS != nil { + return "", nil, fmt.Errorf("render LMCache MP server: LMCache 0.5.3 resp adapter does not support TLS") + } + if binding.Redis.Database != nil { + return "", nil, fmt.Errorf("render LMCache MP server: LMCache 0.5.3 resp adapter does not support database selection") + } + } + + host, portText, ok := splitLMCacheHostPort(strings.TrimSpace(binding.Endpoint)) + if !ok || host == "" || portText == "" { + return "", nil, fmt.Errorf("render LMCache MP server: Redis endpoint %q is not host:port", binding.Endpoint) + } + port, err := strconv.ParseInt(portText, 10, 32) + if err != nil || port < 1 || port > 65535 { + return "", nil, fmt.Errorf("render LMCache MP server: Redis endpoint %q has invalid port %q", binding.Endpoint, portText) + } + payload := map[string]any{"type": "resp", "host": host, "port": port} + raw, err := json.Marshal(payload) + if err != nil { + return "", nil, fmt.Errorf("render LMCache MP server: marshal RESP adapter: %w", err) + } + + var env []corev1.EnvVar + if binding.Redis != nil && binding.Redis.Authentication != nil { + auth := binding.Redis.Authentication + if auth.Username != nil { + env = append(env, corev1.EnvVar{Name: lmCacheRESPUsernameEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: auth.Username.DeepCopy()}}) + } + env = append(env, corev1.EnvVar{Name: lmCacheRESPPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: auth.Password.DeepCopy()}}) + } + return string(raw), env, nil +} + +func quantityAsGiB(q resource.Quantity) (string, error) { + bytes := q.Value() + if bytes <= 0 { + return "", fmt.Errorf("render LMCache MP server: l1Capacity must be greater than zero") + } + const gib = int64(1 << 30) + whole := bytes / gib + remainder := bytes % gib + if remainder == 0 { + return strconv.FormatInt(whole, 10), nil + } + // LMCache parses this argument as float GB and multiplies by 2^30. Keep + // enough decimal precision to preserve ordinary Kubernetes quantities. + value := float64(bytes) / float64(gib) + return strconv.FormatFloat(value, 'f', -1, 64), nil +} + +func lmCacheMPWireIsOurs(pod *corev1.PodSpec) bool { + if pod == nil { + return false + } + c := findContainerByName(pod.InitContainers, lmCacheMPServerContainerName) + if c == nil { + return false + } + for i := range c.Env { + if c.Env[i].Name == lmCacheMPServerManagedEnv && c.Env[i].Value == lmCacheMPServerManagedValue { + return true + } + } + return false +} + +func findContainerByName(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go new file mode 100644 index 00000000..6ace3ff0 --- /dev/null +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go @@ -0,0 +1,259 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +const testLMCacheServerImage = "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func testLMCacheMPConfig() lmCacheMPServerConfig { + return lmCacheMPServerConfig{ + Image: testLMCacheServerImage, + Port: 6500, + ChunkSizeTokens: 256, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 3, + WriteClientConfig: true, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("5Gi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("6Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + }, + }, + } +} + +func TestRenderLMCachePodLocalServerGolden(t *testing.T) { + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: "engine", + Image: "engine:not-connector-owned", + Args: []string{"--model", "gemma"}, + Env: []corev1.EnvVar{{Name: "KEEP", Value: "yes"}}, + VolumeMounts: []corev1.VolumeMount{{Name: "models", MountPath: "/models"}}, + }}} + cfg := testLMCacheMPConfig() + cfg.Binding = &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: "redis.cache.svc:6379"} + + path, err := renderLMCachePodLocalServer(pod, "engine", cfg) + if err != nil { + t.Fatalf("renderLMCachePodLocalServer: %v", err) + } + if path != lmCacheMPConfigFilePath { + t.Fatalf("config path = %q, want %q", path, lmCacheMPConfigFilePath) + } + if got := pod.Containers[0].Image; got != "engine:not-connector-owned" { + t.Fatalf("engine image changed to %q", got) + } + if got := pod.Containers[0].Args; !reflect.DeepEqual(got, []string{"--model", "gemma"}) { + t.Fatalf("common renderer changed engine args: %v", got) + } + + server := findContainerByName(pod.InitContainers, lmCacheMPServerContainerName) + if server == nil { + t.Fatalf("MP server missing: %+v", pod.InitContainers) + } + if server.Image != testLMCacheServerImage || server.Image == pod.Containers[0].Image { + t.Fatalf("server image = %q, engine image = %q", server.Image, pod.Containers[0].Image) + } + if server.RestartPolicy == nil || *server.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Fatalf("server is not a native sidecar: %v", server.RestartPolicy) + } + joined := strings.Join(append(server.Command, server.Args...), " ") + for _, want := range []string{ + "lmcache server", "--host 127.0.0.1", "--port 6500", + "--http-port 8080", "--chunk-size 256", "--l1-size-gb 4", + "--max-workers 3", `{"host":"redis.cache.svc","port":6379,"type":"resp"}`, + } { + if !strings.Contains(joined, want) { + t.Fatalf("server command missing %q: %s", want, joined) + } + } + if strings.Contains(joined, "python3 -m") { + t.Fatalf("server uses unsupported module entrypoint: %s", joined) + } + if server.StartupProbe == nil || server.ReadinessProbe == nil || server.LivenessProbe == nil { + t.Fatalf("HTTP probes incomplete: startup=%+v readiness=%+v liveness=%+v", server.StartupProbe, server.ReadinessProbe, server.LivenessProbe) + } + for _, probe := range []*corev1.Probe{server.StartupProbe, server.ReadinessProbe, server.LivenessProbe} { + if probe.HTTPGet == nil || probe.HTTPGet.Path != lmCacheMPHealthPath || probe.HTTPGet.Port.StrVal != lmCacheMPHTTPPortName { + t.Fatalf("probe is not %s on named HTTP port: %+v", lmCacheMPHealthPath, probe) + } + } + if got := server.Resources.Limits[corev1.ResourceEphemeralStorage]; got.Cmp(resource.MustParse("1Gi")) != 0 { + t.Fatalf("ephemeral-storage limit = %s, want 1Gi", got.String()) + } + if server.SecurityContext == nil || server.SecurityContext.AllowPrivilegeEscalation == nil || *server.SecurityContext.AllowPrivilegeEscalation { + t.Fatalf("server security context allows privilege escalation: %+v", server.SecurityContext) + } + if len(server.SecurityContext.Capabilities.Drop) != 1 || server.SecurityContext.Capabilities.Drop[0] != "ALL" { + t.Fatalf("server capabilities = %+v, want drop ALL", server.SecurityContext.Capabilities) + } + if !hasNamedContainerPort(server.Ports, lmCacheMPServerPortName, 6500) || !hasNamedContainerPort(server.Ports, lmCacheMPHTTPPortName, 8080) { + t.Fatalf("server ports = %+v", server.Ports) + } + shm := findVolume(pod.Volumes, lmCacheMPShmVolumeName) + if shm == nil || shm.EmptyDir == nil || shm.EmptyDir.Medium != corev1.StorageMediumMemory || shm.EmptyDir.SizeLimit == nil || shm.EmptyDir.SizeLimit.Cmp(resource.MustParse("4Gi")) != 0 { + t.Fatalf("shared-memory volume = %+v, want bounded 4Gi tmpfs", shm) + } + if findVolume(pod.Volumes, lmCacheMPConfigVolumeName) == nil { + t.Fatalf("client config volume missing: %+v", pod.Volumes) + } +} + +func TestRenderLMCachePodLocalServerSecretAuthUsesEnvReferences(t *testing.T) { + cfg := testLMCacheMPConfig() + cfg.Binding = &backendadapter.Binding{ + Protocol: backendadapter.ProtocolRESP, + Endpoint: "redis.example:6379", + Redis: &backendadapter.RedisBinding{Authentication: &cachev1alpha1.RedisAuthenticationSpec{ + Username: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "username"}, + Password: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "password"}, + }}, + } + pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine", Image: "engine:test"}}} + if _, err := renderLMCachePodLocalServer(pod, "engine", cfg); err != nil { + t.Fatalf("renderLMCachePodLocalServer: %v", err) + } + server := findContainerByName(pod.InitContainers, lmCacheMPServerContainerName) + if server == nil { + t.Fatal("MP server missing") + } + joined := strings.Join(append(server.Command, server.Args...), " ") + if strings.Contains(joined, "redis-auth") || strings.Contains(joined, "password") || strings.Contains(joined, "username") { + t.Fatalf("secret selector leaked into process arguments: %s", joined) + } + assertSecretEnv := func(name, secret, key string) { + t.Helper() + for i := range server.Env { + if server.Env[i].Name == name { + ref := server.Env[i].ValueFrom + if ref == nil || ref.SecretKeyRef == nil || ref.SecretKeyRef.Name != secret || ref.SecretKeyRef.Key != key { + t.Fatalf("%s = %+v, want %s/%s SecretKeyRef", name, server.Env[i], secret, key) + } + return + } + } + t.Fatalf("%s missing", name) + } + assertSecretEnv(lmCacheRESPUsernameEnv, "redis-auth", "username") + assertSecretEnv(lmCacheRESPPasswordEnv, "redis-auth", "password") +} + +func TestRenderLMCachePodLocalServerIdempotent(t *testing.T) { + cfg := testLMCacheMPConfig() + pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine", Image: "engine:test"}}} + if _, err := renderLMCachePodLocalServer(pod, "engine", cfg); err != nil { + t.Fatalf("first render: %v", err) + } + cfg.MaxWorkers = 7 + if _, err := renderLMCachePodLocalServer(pod, "engine", cfg); err != nil { + t.Fatalf("second render: %v", err) + } + if len(pod.InitContainers) != 1 || len(pod.Volumes) != 2 { + t.Fatalf("re-render duplicated wire: init=%d volumes=%d", len(pod.InitContainers), len(pod.Volumes)) + } + joined := strings.Join(pod.InitContainers[0].Args, " ") + if !strings.Contains(joined, "--max-workers 7") || strings.Contains(joined, "--max-workers 3") { + t.Fatalf("re-render did not converge maxWorkers: %s", joined) + } +} + +func TestRenderLMCachePodLocalServerCollisionIsAtomic(t *testing.T) { + tests := []struct { + name string + pod corev1.PodSpec + }{ + { + name: "foreign reserved server", + pod: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine"}}, + InitContainers: []corev1.Container{{Name: lmCacheMPServerContainerName, Image: "operator:test"}}, + }, + }, + { + name: "foreign config volume", + pod: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine"}}, + Volumes: []corev1.Volume{{Name: lmCacheMPConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "foreign"}}}}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + before := tc.pod.DeepCopy() + if _, err := renderLMCachePodLocalServer(&tc.pod, "engine", testLMCacheMPConfig()); err == nil { + t.Fatal("expected collision error") + } + if !reflect.DeepEqual(&tc.pod, before) { + t.Fatalf("failed render mutated pod\nbefore=%+v\nafter=%+v", before, &tc.pod) + } + }) + } +} + +func TestRenderLMCachePodLocalServerReusesWritableShm(t *testing.T) { + pod := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "engine", + VolumeMounts: []corev1.VolumeMount{{ + Name: "engine-shm", MountPath: "/dev/shm", SubPath: "shared", + }}, + }}, + Volumes: []corev1.Volume{{Name: "engine-shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}}}, + } + if _, err := renderLMCachePodLocalServer(pod, "engine", testLMCacheMPConfig()); err != nil { + t.Fatalf("renderLMCachePodLocalServer: %v", err) + } + if findVolume(pod.Volumes, lmCacheMPShmVolumeName) != nil { + t.Fatalf("renderer added a duplicate shm volume: %+v", pod.Volumes) + } + server := findContainerByName(pod.InitContainers, lmCacheMPServerContainerName) + if server == nil || len(server.VolumeMounts) == 0 || server.VolumeMounts[0].Name != "engine-shm" || server.VolumeMounts[0].SubPath != "shared" { + t.Fatalf("server did not mirror engine shm mount: %+v", server) + } +} + +func TestRenderLMCacheMPL2BindingRejectsUnsupportedV053Features(t *testing.T) { + db := int32(1) + for _, tc := range []struct { + name string + redis *backendadapter.RedisBinding + want string + }{ + {name: "TLS", redis: &backendadapter.RedisBinding{TLS: &cachev1alpha1.RemoteStorageTLSSpec{}}, want: "does not support TLS"}, + {name: "database", redis: &backendadapter.RedisBinding{Database: &db}, want: "does not support database"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, err := renderLMCacheMPL2Binding(&backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: "redis:6379", Redis: tc.redis}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error = %v, want %q", err, tc.want) + } + }) + } +} + +func hasNamedContainerPort(ports []corev1.ContainerPort, name string, port int32) bool { + for i := range ports { + if ports[i].Name == name && ports[i].ContainerPort == port { + return true + } + } + return false +} diff --git a/internal/adapters/builtin/runtime/sglang_lmcache.go b/internal/adapters/builtin/runtime/sglang_lmcache.go index f7093bea..414c47b9 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache.go @@ -15,6 +15,13 @@ import ( ) const ( + // sglangLMCacheMPConnectorProfile is the runtime-owner capability contract + // required before the webhook applies the typed PodLocal MP wire. The image + // pipeline owns this declaration; CacheBackend does not inspect or replace + // the engine image. + sglangLMCacheMPConnectorProfile = "sglang-lmcache-mp-v1" + sglangLMCacheMPClientVersion = "0.5.3" + // subscriberHashScheme is the canonical hash-scheme tag the SGLang // subscriber carries. Kept distinct from the runtime id and from vLLM's // "vllm" tag: the cache plane keys the index on (tenant, model, @@ -46,17 +53,19 @@ const ( // sglangLMCacheAdapter wires SGLang engine pods to LMCache for the (SGLang, LMCache) // pair. SGLang drives LMCache in MULTIPROCESS (MP) mode: // -// - InjectEngineConfig renders a node-local MP-worker -// native sidecar + a config-file (mp_host/mp_port) the engine reads via -// --lmcache-config-file. A nil binding is host-only; an optional RESP -// binding offloads to independently selected Redis storage. +// - Typed PodLocal objects use the shared CacheBackend-owned MP-server native +// sidecar + a config file (mp_host/mp_port) the engine reads via +// --lmcache-config-file. A nil binding is L1-only; an optional RESP binding +// offloads to independently selected Redis storage. Topology-less legacy +// objects retain the prior SGLang-specific worker during compatibility. // - It turns LMCache on with // --enable-lmcache + LMCACHE_USE_EXPERIMENTAL (not vLLM's --kv-transfer-config) // and does NOT inject the lm:// LMCACHE_REMOTE_URL env, which MP mode ignores. // See InjectSGLangLMCache. // -// GPU-validated end-to-end; full design: docs/design/sglang-lmcache-mp-mode.md. The -// kvevent-subscriber sidecar rendering is still shared engine-agnostically. +// The legacy SGLang spike was GPU-validated; the typed common-renderer path is +// intentionally not production-claimed until the Phase 3 GPU matrix passes. +// The kvevent-subscriber sidecar rendering remains engine-agnostic. type sglangLMCacheAdapter struct { subscriber SubscriberConfig } @@ -95,6 +104,9 @@ func (sglangLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) boo // InjectEngineConfig renders SGLang's LMCache MP-mode launch surface from a // host-only nil binding or a RESP binding for Redis L2 storage. func (sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { + if cache != nil && cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" { + return injectSGLangLMCachePodLocal(pod, binding, cache) + } endpoint := "" if binding != nil { if binding.Protocol != backendadapter.ProtocolRESP { @@ -105,6 +117,87 @@ func (sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *bac return InjectSGLangLMCache(pod, endpoint, cache) } +// ConnectorRequirement declares the engine-image-owned connector profile used +// by the typed SGLang MP adapter. Admission compares this with Pod annotations; +// it does not infer capability from an image name. +func (sglangLMCacheAdapter) ConnectorRequirement(*cachev1alpha1.CacheBackend) runtimeadapter.LMCacheConnectorRequirement { + return runtimeadapter.LMCacheConnectorRequirement{ + Profile: sglangLMCacheMPConnectorProfile, + ClientVersion: sglangLMCacheMPClientVersion, + } +} + +// ValidateMPEnginePod checks the concrete Pod constraints needed before the +// common renderer runs. Topology and server resource validation remain at the +// CacheBackend admission boundary; this method owns runtime-visible shape. +func (sglangLMCacheAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1alpha1.CacheBackend) error { + if pod == nil { + return fmt.Errorf("SGLang LMCache MP engine pod is nil") + } + if cache == nil || cache.Spec.LMCache == nil { + return fmt.Errorf("SGLang LMCache MP CacheBackend configuration is missing") + } + if cache.Spec.LMCache.Topology != cachev1alpha1.LMCacheTopologyPodLocal { + return fmt.Errorf("SGLang LMCache MP topology %q is not implemented; want %q", + cache.Spec.LMCache.Topology, cachev1alpha1.LMCacheTopologyPodLocal) + } + if cache.Spec.LMCache.PodLocal == nil || cache.Spec.LMCache.PodLocal.Server == nil { + return fmt.Errorf("SGLang LMCache PodLocal server configuration is missing") + } + if _, err := EngineContainerIndexNamed(&pod.Spec, SGLangEngineContainerName); err != nil { + return err + } + if findContainerByName(pod.Spec.InitContainers, sglangMPWorkerContainerName) != nil { + return fmt.Errorf("legacy LMCache MP sidecar %q is present; recreate the Pod from an un-injected template before enabling typed PodLocal", sglangMPWorkerContainerName) + } + return nil +} + +func injectSGLangLMCachePodLocal(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { + if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { + return err + } + lm := cache.Spec.LMCache + if lm == nil || lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal || lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("inject SGLang LMCache MP: typed PodLocal server configuration is required") + } + server := lm.PodLocal.Server + chunkSize := int32(256) + if lm.ChunkSizeTokens != nil { + chunkSize = *lm.ChunkSizeTokens + } + + // Compose the common server and SGLang launch surface on one copy. Although + // the post-render SGLang upserts cannot fail, keeping one commit point makes + // the adapter's atomicity contract explicit and future-proof. + work := pod.DeepCopy() + configPath, err := renderLMCachePodLocalServer(work, SGLangEngineContainerName, lmCacheMPServerConfig{ + Image: server.Image, + Port: server.Port, + ChunkSizeTokens: chunkSize, + L1Capacity: server.L1Capacity, + MaxWorkers: server.MaxWorkers, + Resources: server.Resources, + Binding: binding, + WriteClientConfig: true, + }) + if err != nil { + return err + } + engineIndex, err := EngineContainerIndexNamed(work, SGLangEngineContainerName) + if err != nil { + return err + } + engine := &work.Containers[engineIndex] + engine.Args = UpsertFlag(engine.Args, SGLangEnableLMCacheArg) + engine.Args = UpsertArgPair(engine.Args, SGLangConfigFileArg, configPath) + engine.Env = UpsertEnv(engine.Env, corev1.EnvVar{Name: EnvLMCacheUseExperimental, Value: lmcacheUseExperimentalVal}) + engine.Env = UpsertEnv(engine.Env, corev1.EnvVar{Name: EnvInferenceCacheFailOpen, Value: FailOpenString(cache)}) + + *pod = *work + return nil +} + // InjectRouterConfig is a no-op for LMCache: the topology has no router // component the controller wires. Returning nil keeps the interface contract // satisfied so a Registry caller can blindly invoke both Inject* paths without @@ -188,3 +281,4 @@ func (sglangLMCacheAdapter) EngineContainerName() string { return SGLangEngineCo // Compile-time assertion: the adapter implements the full C5 interface. var _ runtimeadapter.KVCacheRuntimeAdapter = sglangLMCacheAdapter{} +var _ runtimeadapter.LMCacheMPRuntimeAdapter = sglangLMCacheAdapter{} diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index 0b9b6999..fe3be0fe 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -60,6 +60,32 @@ func newSGLangBackend(cfg map[string]string) *cachev1alpha1.CacheBackend { return cb } +func newTypedSGLangMPBackend() *cachev1alpha1.CacheBackend { + chunkSize := int32(256) + return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + ChunkSizeTokens: &chunkSize, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: testLMCacheServerImage, + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + }, + }}, + }, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{}, + }, + } +} + func respBinding(endpoint string) *backendadapter.Binding { return &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: endpoint} } @@ -203,6 +229,77 @@ func TestSGLangCanonicalHostOnlyBindingDoesNotSelectRedis(t *testing.T) { } } +func TestSGLangTypedPodLocalUsesCommonRenderer(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) + mpAdapter, ok := adapter.(runtimeadapter.LMCacheMPRuntimeAdapter) + if !ok { + t.Fatalf("adapter %T does not implement LMCacheMPRuntimeAdapter", adapter) + } + requirement := mpAdapter.ConnectorRequirement(newTypedSGLangMPBackend()) + if requirement.Profile != sglangLMCacheMPConnectorProfile || requirement.ClientVersion != "0.5.3" { + t.Fatalf("connector requirement = %+v", requirement) + } + + cache := newTypedSGLangMPBackend() + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, + }}} + if err := adapter.InjectEngineConfig(pod, respBinding("redis.ns1.svc:6379"), cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + server := findInitContainer(pod.InitContainers, lmCacheMPServerContainerName) + if server == nil { + t.Fatalf("typed MP server missing: %+v", pod.InitContainers) + } + if server.Image != testLMCacheServerImage || server.Image == pod.Containers[0].Image { + t.Fatalf("server image = %q, engine image = %q", server.Image, pod.Containers[0].Image) + } + joined := strings.Join(append(server.Command, server.Args...), " ") + if !strings.Contains(joined, "lmcache server") || strings.Contains(joined, "python3 -m") { + t.Fatalf("typed server entrypoint = %s", joined) + } + engine := pod.Containers[0] + if !containsArg(engine.Args, SGLangEnableLMCacheArg) || !containsArg(engine.Args, SGLangConfigFileArg) { + t.Fatalf("SGLang typed launch args missing: %v", engine.Args) + } + configIndex := -1 + for i := range engine.Args { + if engine.Args[i] == SGLangConfigFileArg { + configIndex = i + break + } + } + if configIndex < 0 || configIndex+1 >= len(engine.Args) || engine.Args[configIndex+1] != lmCacheMPConfigFilePath { + t.Fatalf("SGLang config path = %v, want %q", engine.Args, lmCacheMPConfigFilePath) + } + if findInitContainer(pod.InitContainers, sglangMPWorkerContainerName) != nil { + t.Fatalf("typed path also injected legacy worker: %+v", pod.InitContainers) + } +} + +func TestSGLangValidateTypedMPEnginePod(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + cache := newTypedSGLangMPBackend() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + runtimeadapter.AnnotationLMCacheConnectorProfile: sglangLMCacheMPConnectorProfile, + runtimeadapter.AnnotationLMCacheClientVersion: sglangLMCacheMPClientVersion, + }}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName}}}, + } + if err := runtimeadapter.ValidateConnectorDeclaration(pod, adapter.ConnectorRequirement(cache)); err != nil { + t.Fatalf("ValidateConnectorDeclaration: %v", err) + } + if err := adapter.ValidateMPEnginePod(pod, cache); err != nil { + t.Fatalf("ValidateMPEnginePod: %v", err) + } + + pod.Spec.InitContainers = []corev1.Container{{Name: sglangMPWorkerContainerName}} + if err := adapter.ValidateMPEnginePod(pod, cache); err == nil || !strings.Contains(err.Error(), "legacy") { + t.Fatalf("legacy collision error = %v", err) + } +} + func TestSGLangInjectEngineConfig(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{}) cb := newSGLangBackend(nil) diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_wire.go b/internal/adapters/builtin/runtime/sglang_lmcache_wire.go index 547b9b4b..9ac9237a 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_wire.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_wire.go @@ -191,7 +191,7 @@ func InjectSGLangLMCache(pod *corev1.PodSpec, endpoint string, cache *cachev1alp // Not every mount can be reused — a read-only or projection-backed one breaks // the MP data path at runtime, deep inside LMCache. Reject at admission and // let the webhook fail open. - if err := sglangCheckShmReusable(work.Volumes, *existing); err != nil { + if err := checkLMCacheMPShmReusable(work.Volumes, *existing); err != nil { return err } // Mirror the engine's subPath. Both containers must land on the SAME @@ -349,7 +349,7 @@ func sglangMPWorkerContainer(engineImage string, engineSC *corev1.SecurityContex PeriodSeconds: 3, FailureThreshold: 40, }, - // Restricted-compatible securityContext (see sglangWorkerSecurityContext). This + // Restricted-compatible securityContext (see lmCacheMPServerSecurityContext). This // mutation lands BEFORE Pod Security admission, so a worker that did NOT carry // the container-only Restricted requirements (allowPrivilegeEscalation: false, // drop ALL capabilities) would get the whole engine pod REJECTED in a @@ -358,31 +358,27 @@ func sglangMPWorkerContainer(engineImage string, engineSC *corev1.SecurityContex // carried it over from the RDMA reference manifests; the MP wire moves KV over // CUDA-IPC and /dev/shm, not RDMA, so no capability is needed — and an added // capability is itself a Restricted violation). - SecurityContext: sglangWorkerSecurityContext(engineSC), + SecurityContext: lmCacheMPServerSecurityContext(engineSC), } } -// sglangWorkerSecurityContext builds the MP worker's securityContext so the worker +// lmCacheMPServerSecurityContext builds an MP server's securityContext so it // never turns an admissible engine pod into a Pod-Security-rejected one. // // It always sets the two container-only Restricted requirements — these cannot be -// inherited from the pod, so the worker must carry them itself: +// inherited from the pod, so the server must carry them itself: // - AllowPrivilegeEscalation=false, -// - Capabilities.Drop=[ALL] (the worker needs no capabilities; GPU access is via +// - Capabilities.Drop=[ALL] (the server needs no capabilities; GPU access is via // device files + /dev/shm, not caps). // // It also sets seccompProfile=RuntimeDefault (Restricted-required; harmless, and GPU // workloads run under it). It deliberately does NOT set RunAsNonRoot / RunAsUser / -// ReadOnlyRootFilesystem to fixed values the way the distroless subscriber sidecar -// does: the worker runs the operator's engine/worker image, whose user and writable -// paths this adapter must not override (forcing a UID or a read-only rootfs can break -// CUDA-IPC or the image's own writes). Instead it MIRRORS the engine container's user -// identity (RunAsNonRoot / RunAsUser / RunAsGroup) when the engine sets it — the -// worker defaults to the same image, so the same user is valid, and matching the -// engine keeps the worker exactly as (non-)root as the pod was admitted to be. When -// the engine leaves those unset, they are inherited from the pod securityContext (the -// usual restricted-namespace shape), so the worker inherits the same. -func sglangWorkerSecurityContext(engineSC *corev1.SecurityContext) *corev1.SecurityContext { +// ReadOnlyRootFilesystem to fixed values: the selected LMCache image owns its +// user and writable-path requirements, and forcing a UID or read-only rootfs can +// break CUDA-IPC or image startup. The legacy renderer passes engineSC so its +// same-image worker mirrors the engine identity; the standalone typed renderer +// passes nil and inherits any Pod-level identity instead. +func lmCacheMPServerSecurityContext(engineSC *corev1.SecurityContext) *corev1.SecurityContext { no := false sc := &corev1.SecurityContext{ AllowPrivilegeEscalation: &no, @@ -549,7 +545,7 @@ func adoptContainer(cs []corev1.Container, want corev1.Container, owned bool) ([ continue } if !owned { - return nil, fmt.Errorf("inject engine config: pod already has a container named %q that this adapter did not render; that name is reserved for the LMCache MP worker — rename your container", want.Name) + return nil, fmt.Errorf("inject engine config: pod already has a container named %q that this adapter did not render; that name is reserved for the LMCache MP native sidecar — rename your container", want.Name) } cs[i] = want // our own prior injection — converge on the current render return cs, nil @@ -574,8 +570,8 @@ func adoptVolume(vs []corev1.Volume, want corev1.Volume, owned bool) ([]corev1.V return append(vs, want), nil } -// sglangCheckShmReusable rejects an engine-owned /dev/shm mount the worker cannot -// safely share. The engine and the worker exchange KV through this volume, so it +// checkLMCacheMPShmReusable rejects an engine-owned /dev/shm mount the MP server cannot +// safely share. The engine and the server exchange KV through this volume, so it // must be WRITABLE and both containers must resolve it to the SAME directory — // neither of which the kubelet reports back at admission; getting it wrong surfaces // as a silent no-transfer at runtime, deep inside LMCache. @@ -587,9 +583,9 @@ func adoptVolume(vs []corev1.Volume, want corev1.Volume, owned bool) ([]corev1.V // downwardAPI / projected) AND every in-tree source carrying its own readOnly flag. // Sources with no such flag (emptyDir, hostPath, ephemeral, …) are writable, or // their writability is the operator's to configure, so they pass. -func sglangCheckShmReusable(vs []corev1.Volume, m corev1.VolumeMount) error { +func checkLMCacheMPShmReusable(vs []corev1.Volume, m corev1.VolumeMount) error { if m.ReadOnly { - return fmt.Errorf("inject engine config: engine container mounts %q read-only (volume %q), but the LMCache MP data path writes there — drop readOnly or mount it elsewhere", sglangShmMountPath, m.Name) + return fmt.Errorf("inject engine config: engine container mounts %q read-only (volume %q), but the LMCache MP data path writes there — drop readOnly or mount it elsewhere", lmCacheMPShmMountPath, m.Name) } // subPath is mirrorable (the caller copies it onto the worker's mount); // subPathExpr is NOT: it expands $(VAR) from the mounting CONTAINER's env, and the @@ -598,7 +594,7 @@ func sglangCheckShmReusable(vs []corev1.Volume, m corev1.VolumeMount) error { // containers on different directories is exactly the failure this guard exists to // prevent, so reject rather than guess. if m.SubPathExpr != "" { - return fmt.Errorf("inject engine config: engine container mounts %q with subPathExpr %q (volume %q); the LMCache MP worker cannot reproduce that expansion in its own env — use a literal subPath, or mount %[1]q without it", sglangShmMountPath, m.SubPathExpr, m.Name) + return fmt.Errorf("inject engine config: engine container mounts %q with subPathExpr %q (volume %q); the LMCache MP server cannot reproduce that expansion in its own env — use a literal subPath, or mount %[1]q without it", lmCacheMPShmMountPath, m.SubPathExpr, m.Name) } for i := range vs { if vs[i].Name != m.Name { @@ -667,7 +663,7 @@ func sglangCheckShmReusable(vs []corev1.Volume, m corev1.VolumeMount) error { default: return nil } - return fmt.Errorf("inject engine config: engine container mounts %q from a %s volume (%q) %s, but the LMCache MP data path writes there — use an emptyDir (medium: Memory) instead", sglangShmMountPath, kind, m.Name, why) + return fmt.Errorf("inject engine config: engine container mounts %q from a %s volume (%q) %s, but the LMCache MP data path writes there — use an emptyDir (medium: Memory) instead", lmCacheMPShmMountPath, kind, m.Name, why) } return nil } diff --git a/internal/adapters/builtin/storage/redis.go b/internal/adapters/builtin/storage/redis.go index 99908004..13d80902 100644 --- a/internal/adapters/builtin/storage/redis.go +++ b/internal/adapters/builtin/storage/redis.go @@ -53,6 +53,8 @@ const ( // It matches the provider's 8Gi memory default. redisMaxmemoryDefaultBytes = int64(8) * 1024 * 1024 * 1024 // 8Gi + redisPasswordEnv = "REDIS_PASSWORD" + redisCLIAuthEnv = "REDISCLI_AUTH" ) // ResolveRedisL2Server renders the managed Redis L2 store's container set and the @@ -70,11 +72,13 @@ const ( // CacheBackend), added alongside the wiring that consumes this render — before it // provisions anything. // -// Security posture matches the lm:// lmcache-server this replaces: an -// unauthenticated, non-TLS ClusterIP holding KV blocks, trusted to the in-cluster -// network — any pod that can reach the Service can read, overwrite, or flush -// cached KV. Hardening (a NetworkPolicy scoping access to engine pods, Redis AUTH, -// or TLS) is a follow-up carried at the same posture as the existing server. +// Security posture is explicit: without an authentication binding this is an +// unauthenticated, non-TLS ClusterIP holding KV blocks, so any pod that can +// reach the Service can read, overwrite, or flush cached KV. Secret-backed +// password authentication is supported below and configures both Redis and the +// LMCache RESP client. TLS remains rejected because the pinned LMCache 0.5.3 +// RESP adapter does not implement it; NetworkPolicy scoping is still required +// for production defense in depth. func ResolveRedisL2Server(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { if cache == nil { return nil, nil, fmt.Errorf("resolve redis L2: cache is nil") @@ -129,6 +133,39 @@ func ResolveRedisL2Server(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, * Resources: defaultServerResources(cache), } + // Managed authentication configures BOTH ends of the binding: this Redis + // process requires the Secret-backed password, while the MP renderer maps + // the same selector to LMCACHE_RESP_PASSWORD. The secret value never enters + // the PodSpec or CacheBackend status. The shell expands it only inside the + // container and then explicitly invokes the official image entrypoint, which + // preserves its root-to-redis privilege drop. + if storage := cache.Spec.EffectiveRemoteStorage(); storage != nil && storage.Redis != nil && storage.Redis.Authentication != nil { + redis := storage.Redis + auth := redis.Authentication + if auth.Username != nil { + return nil, nil, fmt.Errorf("resolve redis L2: managed Redis supports password authentication with the default user only") + } + passwordRef := auth.Password.DeepCopy() + container.Env = append(container.Env, + corev1.EnvVar{Name: redisPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: passwordRef}}, + corev1.EnvVar{Name: redisCLIAuthEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: auth.Password.DeepCopy()}}, + ) + const authScript = `set -eu +exec /usr/local/bin/docker-entrypoint.sh "$@" --requirepass "$REDIS_PASSWORD"` + container.Command = []string{"/bin/sh", "-c"} + container.Args = append([]string{authScript, "inference-cache-redis-auth"}, container.Args...) + // REDISCLI_AUTH lets redis-cli authenticate without placing the password + // in the probe command. A TCP-only probe would declare a server Ready even + // if AUTH setup were unusable. + container.ReadinessProbe = &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"redis-cli", "ping"}}}, + InitialDelaySeconds: 3, + PeriodSeconds: 10, + FailureThreshold: 6, + TimeoutSeconds: 2, + } + } + pod := &corev1.PodSpec{ Containers: []corev1.Container{container}, } diff --git a/internal/adapters/builtin/storage/redis_test.go b/internal/adapters/builtin/storage/redis_test.go index bd6fbb44..2eb96af6 100644 --- a/internal/adapters/builtin/storage/redis_test.go +++ b/internal/adapters/builtin/storage/redis_test.go @@ -6,6 +6,7 @@ package storage import ( "strconv" + "strings" "testing" corev1 "k8s.io/api/core/v1" @@ -199,6 +200,61 @@ func TestResolveRedisL2ServerImageOverride(t *testing.T) { } } +func TestResolveRedisL2ServerPasswordAuthentication(t *testing.T) { + cb := newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang") + cb.Spec.RemoteStorage.Redis.Authentication = &cachev1alpha1.RedisAuthenticationSpec{ + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "credential-source"}, + Key: "password", + }, + } + pod, _, err := ResolveRedisL2Server(cb) + if err != nil { + t.Fatalf("ResolveRedisL2Server: %v", err) + } + c := pod.Containers[0] + if len(c.Command) != 2 || c.Command[0] != "/bin/sh" || c.Command[1] != "-c" { + t.Fatalf("authenticated Redis command = %v", c.Command) + } + joined := strings.Join(c.Args, " ") + if !strings.Contains(joined, "/usr/local/bin/docker-entrypoint.sh") || !strings.Contains(joined, `--requirepass "$REDIS_PASSWORD"`) { + t.Fatalf("authenticated Redis args = %s", joined) + } + if strings.Contains(joined, "credential-source") { + t.Fatalf("secret name leaked into args: %s", joined) + } + for _, name := range []string{redisPasswordEnv, redisCLIAuthEnv} { + found := false + for i := range c.Env { + if c.Env[i].Name != name { + continue + } + found = true + ref := c.Env[i].ValueFrom + if ref == nil || ref.SecretKeyRef == nil || ref.SecretKeyRef.Name != "credential-source" || ref.SecretKeyRef.Key != "password" { + t.Fatalf("%s = %+v", name, c.Env[i]) + } + } + if !found { + t.Fatalf("%s missing", name) + } + } + if c.ReadinessProbe == nil || c.ReadinessProbe.Exec == nil || !strings.Contains(strings.Join(c.ReadinessProbe.Exec.Command, " "), "redis-cli ping") { + t.Fatalf("authenticated readiness probe = %+v", c.ReadinessProbe) + } +} + +func TestResolveRedisL2ServerRejectsManagedACLUsername(t *testing.T) { + cb := newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang") + cb.Spec.RemoteStorage.Redis.Authentication = &cachev1alpha1.RedisAuthenticationSpec{ + Username: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "username"}, + Password: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "password"}, + } + if _, _, err := ResolveRedisL2Server(cb); err == nil || !strings.Contains(err.Error(), "default user only") { + t.Fatalf("error = %v", err) + } +} + func TestResolveRedisL2ServerMaxmemory(t *testing.T) { // ~80% via base - base/5 — integer division rounds slightly UP of exact 80% // (exact only when base is divisible by 5), favoring dataset capacity at the cost diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go new file mode 100644 index 00000000..c94d1a00 --- /dev/null +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -0,0 +1,236 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" +) + +const ( + conditionTypeConnectorReady = "ConnectorReady" + conditionTypeRemoteStorageReady = "RemoteStorageReady" + + reasonConnectorReady = "ConnectorReady" + reasonConnectorUnverified = "ConnectorCapabilityUnverified" + reasonNoEnginePods = "NoEnginePods" + reasonMPServersNotReady = "MPServersNotReady" + reasonRemoteStorageReady = "RemoteStorageReady" + reasonRemoteStorageAbsent = "RemoteStorageNotConfigured" + reasonRemoteStoragePending = "RemoteStoragePending" + reasonRemoteStorageUnavailable = "RemoteStorageUnavailable" + + lmCacheMPServerStatusContainerName = "lmcache-mp-server" +) + +func isTypedLMCachePodLocal(backend *cachev1alpha1.CacheBackend) bool { + return backend != nil && + backend.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && + backend.Spec.LMCache != nil && + backend.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal +} + +// refreshLMCacheMPConnectorStatus projects PodLocal server health independently +// from the optional remote L3. Native sidecars report their state under +// status.initContainerStatuses, not containerStatuses. +// +// Like matchedEnginePods, this is a bounded-cadence observation rather than a +// cluster-wide Pod watch. List/patch errors preserve the prior verdict and are +// fail-soft so connector observability cannot block normal reconciliation. +func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend) { + if !isTypedLMCachePodLocal(backend) { + if backend.Status.Connector == nil && meta.FindStatusCondition(backend.Status.Conditions, conditionTypeConnectorReady) == nil { + return + } + before := backend.DeepCopy() + backend.Status.Connector = nil + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeConnectorReady) + if err := r.Status().Patch(ctx, backend, client.MergeFrom(before)); err != nil { + backend.Status = before.Status + log.FromContext(ctx).V(1).Info("LMCache MP connector status clear skipped: patch failed", + "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + } + return + } + + reader := client.Reader(r.APIReader) + if reader == nil { + reader = r.Client + } + var pods corev1.PodList + sel := backend.Spec.EngineSelector + if sel != nil && len(sel.MatchLabels) > 0 { + if err := reader.List(ctx, &pods, + client.InNamespace(backend.Namespace), + client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(sel.MatchLabels)}, + ); err != nil { + log.FromContext(ctx).V(1).Info("LMCache MP connector status refresh skipped: pod list failed", + "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + return + } + } + + var matched, verified, readyEngines, desiredServers, readyServers, covered int32 + wantInjectedBy := backend.Namespace + "/" + backend.Name + wantUID := string(backend.UID) + for i := range pods.Items { + pod := &pods.Items[i] + if pod.DeletionTimestamp != nil || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + matched++ + desiredServers++ + annotations := pod.GetAnnotations() + injected := annotations[enginebinding.AnnotationInjectedBy] == wantInjectedBy && + wantUID != "" && annotations[enginebinding.AnnotationInjectedByUID] == wantUID && + annotations[enginebinding.AnnotationInjectedGeneration] == strconv.FormatInt(backend.Generation, 10) + if injected { + verified++ + } + serverReady := injected && nativeSidecarReady(pod.Status.InitContainerStatuses, lmCacheMPServerStatusContainerName) + if serverReady { + readyServers++ + covered++ + if podReady(pod) { + readyEngines++ + } + } + } + + connector := &cachev1alpha1.CacheBackendConnectorStatus{ + Mode: cachev1alpha1.LMCacheConnectorModeMultiprocess, + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + MatchedEnginePods: matched, + ReadyEnginePods: readyEngines, + DesiredServers: desiredServers, + ReadyServers: readyServers, + CoveredEnginePods: covered, + UncoveredEnginePods: matched - covered, + } + status, reason, message := metav1.ConditionFalse, reasonMPServersNotReady, + fmt.Sprintf("%d/%d selected engine Pods have a Ready LMCache MP native sidecar; %d engine Pods are Ready with the connector", readyServers, desiredServers, readyEngines) + if matched == 0 { + reason = reasonNoEnginePods + message = "no active engine Pods match spec.engineSelector" + } else if verified != matched { + status = metav1.ConditionUnknown + reason = reasonConnectorUnverified + message = fmt.Sprintf("%d/%d selected engine Pods carry the webhook-authenticated connector declaration; unverified Pods are left un-injected", verified, matched) + } else if readyServers == desiredServers && readyEngines == matched { + status = metav1.ConditionTrue + reason = reasonConnectorReady + message = fmt.Sprintf("all %d selected engine Pods have a Ready LMCache MP server and connector", matched) + } + + before := backend.DeepCopy() + backend.Status.Connector = connector + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeConnectorReady, + Status: status, + Reason: reason, + Message: message, + ObservedGeneration: backend.Generation, + }) + if err := r.Status().Patch(ctx, backend, client.MergeFrom(before)); err != nil { + backend.Status = before.Status + log.FromContext(ctx).V(1).Info("LMCache MP connector status refresh skipped: patch failed", + "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) + } +} + +func nativeSidecarReady(statuses []corev1.ContainerStatus, name string) bool { + for i := range statuses { + if statuses[i].Name == name { + return statuses[i].Ready && statuses[i].State.Running != nil + } + } + return false +} + +func podReady(pod *corev1.Pod) bool { + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type == corev1.PodReady { + return pod.Status.Conditions[i].Status == corev1.ConditionTrue + } + } + return false +} + +// lmCacheMPReadyBase aggregates the required PodLocal connector and optional +// remote L3 without hiding either component's dedicated condition. The +// connector always gates Ready because SGLang cannot run with +// --enable-lmcache when its co-scheduled MP server is unavailable. Remote +// storage gates Ready only when the operator explicitly selects fail-closed; +// in the default fail-open mode the MP server can continue serving from L1. +// +// The connector condition is refreshed after the main reconcile dispatch, so +// a newly created or newly updated object can spend one reconcile at Unknown +// before the Pod observation is folded into Ready. That is preferable to +// briefly publishing Ready=True from the Redis workload alone. +func lmCacheMPReadyBase( + backend *cachev1alpha1.CacheBackend, + remoteStatus metav1.ConditionStatus, + remoteReason, remoteMessage string, +) (metav1.ConditionStatus, string, string) { + if !isTypedLMCachePodLocal(backend) { + return remoteStatus, remoteReason, remoteMessage + } + + connector := meta.FindStatusCondition(backend.Status.Conditions, conditionTypeConnectorReady) + if connector == nil || connector.ObservedGeneration != backend.Generation { + return metav1.ConditionUnknown, reasonConnectorUnverified, + "connector health for the current CacheBackend generation has not been observed yet" + } + if connector.Status != metav1.ConditionTrue { + return connector.Status, connector.Reason, connector.Message + } + + storage := backend.Spec.EffectiveRemoteStorage() + if storage != nil && !cachev1alpha1.IntegrationFailOpen(backend.Spec.Integration) && remoteStatus != metav1.ConditionTrue { + return remoteStatus, remoteReason, remoteMessage + } + if storage != nil && remoteStatus != metav1.ConditionTrue { + return metav1.ConditionTrue, reasonConnectorReady, + "the Pod-local MP connector is ready; remote storage is degraded but does not gate Ready while failOpen is true" + } + return metav1.ConditionTrue, reasonConnectorReady, connector.Message +} + +func setRemoteStorageStatus(backend *cachev1alpha1.CacheBackend, endpoint string, ready metav1.ConditionStatus, reason, message string, observedGeneration int64) { + if !isTypedLMCachePodLocal(backend) { + backend.Status.RemoteStorage = nil + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeRemoteStorageReady) + return + } + storage := backend.Spec.EffectiveRemoteStorage() + if storage == nil { + backend.Status.RemoteStorage = nil + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeRemoteStorageReady) + return + } + backend.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: storage.Provider, + Endpoint: endpoint, + Ready: ready, + } + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeRemoteStorageReady, + Status: ready, + Reason: reason, + Message: message, + ObservedGeneration: observedGeneration, + }) +} diff --git a/internal/controller/cachebackend_lmcache_mp_status_test.go b/internal/controller/cachebackend_lmcache_mp_status_test.go new file mode 100644 index 00000000..54a305b6 --- /dev/null +++ b/internal/controller/cachebackend_lmcache_mp_status_test.go @@ -0,0 +1,336 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" +) + +func typedMPStatusBackend() *cachev1alpha1.CacheBackend { + return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1", UID: types.UID("cache-uid"), Generation: 3}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeLMCache, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + "app": "sglang", + }}, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("1Gi"), + MaxWorkers: 1, + }}, + }, + }, + } +} + +func typedMPStatusPod(name string, injected, serverReady, engineReady bool) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns1", Labels: map[string]string{"app": "sglang"}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + if injected { + pod.Annotations = map[string]string{ + enginebinding.AnnotationInjectedBy: "ns1/cache", + enginebinding.AnnotationInjectedByUID: "cache-uid", + enginebinding.AnnotationInjectedGeneration: "3", + } + } + if serverReady { + pod.Status.InitContainerStatuses = []corev1.ContainerStatus{{ + Name: lmCacheMPServerStatusContainerName, + Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + } + if engineReady { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return pod +} + +func TestRefreshLMCacheMPConnectorStatusTransitions(t *testing.T) { + ctx := context.Background() + scheme := newScheme(t) + backend := typedMPStatusBackend() + ready := typedMPStatusPod("ready", true, true, true) + uncovered := typedMPStatusPod("uncovered", false, false, true) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &corev1.Pod{}). + WithObjects(backend, ready, uncovered).Build() + r := &CacheBackendReconciler{Client: c, APIReader: c} + + r.refreshLMCacheMPConnectorStatus(ctx, backend) + var got cachev1alpha1.CacheBackend + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get backend: %v", err) + } + status := got.Status.Connector + if status == nil || status.MatchedEnginePods != 2 || status.DesiredServers != 2 || status.ReadyServers != 1 || + status.ReadyEnginePods != 1 || status.CoveredEnginePods != 1 || status.UncoveredEnginePods != 1 { + t.Fatalf("connector status = %+v", status) + } + cond := meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) + if cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonConnectorUnverified { + t.Fatalf("ConnectorReady = %+v", cond) + } + + // Simulate the previously uncovered Pod being recreated through the webhook + // and the native sidecar becoming Ready. The next bounded-cadence refresh + // transitions the connector independently of remote storage. + var live corev1.Pod + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "uncovered"}, &live); err != nil { + t.Fatalf("get uncovered pod: %v", err) + } + live.Annotations = map[string]string{ + enginebinding.AnnotationInjectedBy: "ns1/cache", + enginebinding.AnnotationInjectedByUID: "cache-uid", + enginebinding.AnnotationInjectedGeneration: "3", + } + if err := c.Update(ctx, &live); err != nil { + t.Fatalf("update pod annotations: %v", err) + } + live.Status.InitContainerStatuses = []corev1.ContainerStatus{{ + Name: lmCacheMPServerStatusContainerName, + Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + live.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := c.Status().Update(ctx, &live); err != nil { + t.Fatalf("update pod status: %v", err) + } + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, backend); err != nil { + t.Fatalf("refresh backend object: %v", err) + } + r.refreshLMCacheMPConnectorStatus(ctx, backend) + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get transitioned backend: %v", err) + } + cond = meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) + if cond == nil || cond.Status != metav1.ConditionTrue || cond.Reason != reasonConnectorReady { + t.Fatalf("ConnectorReady after recovery = %+v", cond) + } + if got.Status.Connector.ReadyServers != 2 || got.Status.Connector.UncoveredEnginePods != 0 { + t.Fatalf("connector status after recovery = %+v", got.Status.Connector) + } + + // A CacheBackend spec generation change does not mutate existing Pods. + // Their generation stamp makes the old render explicitly unverified until + // the inference owner recreates/rolls them through admission. + got.Generation = 4 + if err := c.Update(ctx, &got); err != nil { + t.Fatalf("advance backend generation: %v", err) + } + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, backend); err != nil { + t.Fatalf("get generation-4 backend: %v", err) + } + r.refreshLMCacheMPConnectorStatus(ctx, backend) + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get stale-wiring backend: %v", err) + } + cond = meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) + if cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonConnectorUnverified { + t.Fatalf("ConnectorReady after spec generation change = %+v", cond) + } +} + +func TestSetRemoteStorageStatusIndependentFromConnector(t *testing.T) { + backend := typedMPStatusBackend() + backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + backend.Status.Connector = &cachev1alpha1.CacheBackendConnectorStatus{ReadyServers: 0, DesiredServers: 1} + setRemoteStorageStatus(backend, "redis.example:6379", metav1.ConditionTrue, reasonRemoteStorageReady, "ready", backend.Generation) + if backend.Status.RemoteStorage == nil || backend.Status.RemoteStorage.Ready != metav1.ConditionTrue || backend.Status.RemoteStorage.Endpoint != "redis.example:6379" { + t.Fatalf("remote storage status = %+v", backend.Status.RemoteStorage) + } + if backend.Status.Connector.ReadyServers != 0 { + t.Fatalf("remote status update changed connector: %+v", backend.Status.Connector) + } + cond := meta.FindStatusCondition(backend.Status.Conditions, conditionTypeRemoteStorageReady) + if cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("RemoteStorageReady = %+v", cond) + } +} + +func TestLMCacheMPReadyBase(t *testing.T) { + falseV := false + tests := []struct { + name string + connector *metav1.Condition + remoteStatus metav1.ConditionStatus + failClosed bool + wantStatus metav1.ConditionStatus + wantReason string + }{ + { + name: "current connector observation is required", + remoteStatus: metav1.ConditionTrue, + wantStatus: metav1.ConditionUnknown, + wantReason: reasonConnectorUnverified, + }, + { + name: "connector failure always gates ready", + connector: &metav1.Condition{ + Type: conditionTypeConnectorReady, Status: metav1.ConditionFalse, + Reason: reasonMPServersNotReady, Message: "server down", ObservedGeneration: 3, + }, + remoteStatus: metav1.ConditionTrue, + wantStatus: metav1.ConditionFalse, + wantReason: reasonMPServersNotReady, + }, + { + name: "remote failure is independent under default fail open", + connector: &metav1.Condition{ + Type: conditionTypeConnectorReady, Status: metav1.ConditionTrue, + Reason: reasonConnectorReady, Message: "connector ready", ObservedGeneration: 3, + }, + remoteStatus: metav1.ConditionFalse, + wantStatus: metav1.ConditionTrue, + wantReason: reasonConnectorReady, + }, + { + name: "remote failure gates explicit fail closed", + connector: &metav1.Condition{ + Type: conditionTypeConnectorReady, Status: metav1.ConditionTrue, + Reason: reasonConnectorReady, Message: "connector ready", ObservedGeneration: 3, + }, + remoteStatus: metav1.ConditionFalse, + failClosed: true, + wantStatus: metav1.ConditionFalse, + wantReason: reasonRemoteStorageUnavailable, + }, + { + name: "stale connector generation stays unknown", + connector: &metav1.Condition{ + Type: conditionTypeConnectorReady, Status: metav1.ConditionTrue, + Reason: reasonConnectorReady, Message: "old render", ObservedGeneration: 2, + }, + remoteStatus: metav1.ConditionTrue, + wantStatus: metav1.ConditionUnknown, + wantReason: reasonConnectorUnverified, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + backend := typedMPStatusBackend() + backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + if tc.failClosed { + backend.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: &falseV} + } + if tc.connector != nil { + meta.SetStatusCondition(&backend.Status.Conditions, *tc.connector) + } + + gotStatus, gotReason, _ := lmCacheMPReadyBase( + backend, tc.remoteStatus, reasonRemoteStorageUnavailable, "Redis unavailable", + ) + if gotStatus != tc.wantStatus || gotReason != tc.wantReason { + t.Fatalf("ready base = (%s, %q), want (%s, %q)", gotStatus, gotReason, tc.wantStatus, tc.wantReason) + } + }) + } +} + +func TestTypedMPManagedRedisRestartDoesNotCascadeEngineDeployment(t *testing.T) { + backend := typedMPStatusBackend() + backend.Status.ObservedServerInstance = "legacy-remote-instance-latch" + backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{Image: "redis:7.4-alpine"}, + } + controller := true + engineDep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "sglang-engine", Namespace: "ns1", UID: "engine-dep-uid"}, + Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "sglang"}}}}, + } + engineRS := &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{ + Name: "sglang-engine-rs", Namespace: "ns1", UID: "engine-rs-uid", + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: engineDep.Name, UID: engineDep.UID, Controller: &controller, + }}, + }} + enginePod := typedMPStatusPod("sglang-engine-pod", true, true, true) + enginePod.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "ReplicaSet", Name: engineRS.Name, UID: engineRS.UID, Controller: &controller, + }} + + r := newReconciler(newScheme(t), backend, engineDep, engineRS, enginePod) + reconcile(t, r, backend.Name, backend.Namespace) + + var gotDep appsv1.Deployment + if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns1", Name: engineDep.Name}, &gotDep); err != nil { + t.Fatalf("get engine Deployment: %v", err) + } + if got := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != "" { + t.Fatalf("Redis lifecycle cascaded engine Deployment with trigger %q", got) + } + var gotBackend cachev1alpha1.CacheBackend + if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns1", Name: backend.Name}, &gotBackend); err != nil { + t.Fatalf("get CacheBackend: %v", err) + } + if gotBackend.Status.ObservedServerInstance != "" { + t.Fatalf("typed MP retained legacy remote instance latch %q", gotBackend.Status.ObservedServerInstance) + } +} + +func TestMPConditionEventsFireOnTransitionOrObservedGeneration(t *testing.T) { + backend := typedMPStatusBackend() + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeConnectorReady, Status: metav1.ConditionFalse, + Reason: reasonMPServersNotReady, Message: "server restarting", ObservedGeneration: 3, + }) + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeRemoteStorageReady, Status: metav1.ConditionTrue, + Reason: reasonRemoteStorageReady, Message: "Redis ready", ObservedGeneration: 3, + }) + recorder := events.NewFakeRecorder(8) + r := &CacheBackendReconciler{Recorder: recorder} + + r.emitTransitionEvents(backend, stateSnapshot{}) + got := strings.Join(drainEvents(recorder), "\n") + if !strings.Contains(got, reasonMPServersNotReady) || !strings.Contains(got, reasonRemoteStorageReady) { + t.Fatalf("transition events = %q", got) + } + + before := snapshotState(backend) + r.emitTransitionEvents(backend, before) + if events := drainEvents(recorder); len(events) != 0 { + t.Fatalf("steady-state events = %v", events) + } + condition := meta.FindStatusCondition(backend.Status.Conditions, conditionTypeConnectorReady) + condition.ObservedGeneration = 4 + r.emitTransitionEvents(backend, before) + got = strings.Join(drainEvents(recorder), "\n") + if !strings.Contains(got, reasonMPServersNotReady) { + t.Fatalf("generation-change events = %q", got) + } +} diff --git a/internal/controller/cachebackend_managed.go b/internal/controller/cachebackend_managed.go index f1fcaa46..adfe49a3 100644 --- a/internal/controller/cachebackend_managed.go +++ b/internal/controller/cachebackend_managed.go @@ -16,6 +16,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "time" ) // reconcileManaged renders the cache-server PodSpec + Service via the runtime @@ -126,12 +127,29 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo // recovery from a cache-server outage. A non-zero cascadeWait means // the rate-limit window suppressed the cascade; honor it on the // requeue so we retry exactly at the boundary. - cascadeWait := r.reconcileServerInstance(ctx, logger, backend) + cascadeWait := time.Duration(0) + if isTypedLMCachePodLocal(backend) { + // In the typed MP hierarchy this managed workload is Redis L3, not the + // engine's connector endpoint. Redis failure/recovery belongs to the MP + // server's L2 adapter; rolling every engine on a Redis restart creates + // serving disruption without repairing that adapter. MP native-sidecar + // restarts are observed separately through ConnectorReady. + r.clearServerInstanceLatchShadow(backend) + if backend.Status.ObservedServerInstance != "" { + if clearErr := r.patchStatus(ctx, backend, func() { backend.Status.ObservedServerInstance = "" }); clearErr != nil && statusErr == nil { + statusErr = clearErr + } + } + } else { + cascadeWait = r.reconcileServerInstance(ctx, logger, backend) + } if cascadeWait > 0 && (requeueAfter == 0 || cascadeWait < requeueAfter) { requeueAfter = cascadeWait } - // Schedule an unconditional periodic re-poll of the cache-server - // pod set on managed backends. Reason: an in-place container + // Schedule an unconditional periodic health re-poll on managed backends. + // Typed MP uses it to refresh engine-Pod native-sidecar health; legacy + // server-backed paths use it to observe the cache-server pod set. For the + // latter, an in-place container // restart (kubelet respawning a crashed cache-server container // without bumping pod.UID) does NOT change owned-Deployment status // counts, and the controller deliberately does not watch Pods diff --git a/internal/controller/cachebackend_reconciler.go b/internal/controller/cachebackend_reconciler.go index ecd4b66d..1e637b85 100644 --- a/internal/controller/cachebackend_reconciler.go +++ b/internal/controller/cachebackend_reconciler.go @@ -197,6 +197,9 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request // transient List/Patch errors so it never escalates a transient // apiserver hiccup into a Reconcile error. matchedRefresh := r.refreshMatchedEnginePods(ctx, &backend) + // Typed LMCache PodLocal health comes from the injected native sidecars in + // engine Pod status, independently from managed/external Redis readiness. + r.refreshLMCacheMPConnectorStatus(ctx, &backend) // Self-requeue when there's matchedEnginePods work to keep doing on // the next tick: // diff --git a/internal/controller/cachebackend_serverless.go b/internal/controller/cachebackend_serverless.go index b0b90a9e..0af5b8c3 100644 --- a/internal/controller/cachebackend_serverless.go +++ b/internal/controller/cachebackend_serverless.go @@ -18,11 +18,11 @@ import ( ) // reconcileExternal mirrors an externally owned backend's configured endpoint -// to status and marks the backend Ready: there is no Service to wait on, so -// admission acceptance of spec.remoteStorage.endpoint is the only readiness -// signal the controller has. The Ready condition flips to True in lock step so -// the Ready printcolumn (kubectl get cb) reflects the accepted endpoint for -// externally owned resources that admission has already accepted. +// to status. For legacy backends, admission acceptance of the endpoint remains +// the only readiness signal because there is no Service to wait on. Typed +// PodLocal MP additionally aggregates the independently observed connector; +// endpoint acceptance alone must not report a missing/unhealthy native sidecar +// as Ready. // // Three terminal states, each driven by the SAME shape rule the // validating webhook applies on CREATE/UPDATE — so the reconciler is @@ -116,11 +116,23 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend readyMsg = "External endpoint accepted; controller does not provision cache pods for External backends" } + remoteReason := readyReason + remoteMessage := readyMsg + if readyStatus == metav1.ConditionTrue { + remoteReason = reasonRemoteStorageReady + remoteMessage = "external remote-storage binding was accepted; reachability is owned by the operator" + } + remoteStatus := readyStatus + setRemoteStorageStatus(backend, endpoint, remoteStatus, remoteReason, remoteMessage, backend.Generation) + if isTypedLMCachePodLocal(backend) { + readyStatus, readyReason, readyMsg = lmCacheMPReadyBase(backend, remoteStatus, remoteReason, remoteMessage) + } + progressingStatus, progressingReason, progressingMessage := progressingFromReady(readyStatus, readyReason, readyMsg) meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ Type: conditionTypeProgressing, - Status: metav1.ConditionFalse, - Reason: readyReason, - Message: "External backends complete admission immediately", + Status: progressingStatus, + Reason: progressingReason, + Message: progressingMessage, ObservedGeneration: backend.Generation, }) meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ @@ -213,15 +225,19 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen // so also bypass any sticky NoKVEventsObserved carried over from the prior // mode — otherwise the flip would inherit that timed-out verdict and stay // Degraded despite the fresh window. - gate := evaluateKVEventReadiness(backend, metav1.ConditionTrue, - activeReason, - activeMessage, + eventsOnly := backend.Spec.IsEventsOnly() + baseStatus, baseReason, baseMessage := metav1.ConditionTrue, activeReason, activeMessage + if !eventsOnly { + baseStatus, baseReason, baseMessage = lmCacheMPReadyBase(backend, baseStatus, baseReason, baseMessage) + } + gate := evaluateKVEventReadiness(backend, baseStatus, + baseReason, + baseMessage, anchor, now, transitionedFromServerMode) kernelVerdict := kernelHealthVerdict{} engineCompatMsg := "" engineCompatObserved := false previousEngineIncompatible := false - eventsOnly := backend.Spec.IsEventsOnly() if !eventsOnly { kernelReader := client.Reader(r.APIReader) if kernelReader == nil { @@ -247,6 +263,13 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen backend.Status.Endpoint = "" backend.Status.ObservedServerInstance = "" backend.Status.ObservedGeneration = backend.Generation + if eventsOnly { + backend.Status.RemoteStorage = nil + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeRemoteStorageReady) + } else { + setRemoteStorageStatus(backend, "", metav1.ConditionTrue, reasonRemoteStorageAbsent, + "no remote storage is configured; the MP server operates with Pod-local L1 only", backend.Generation) + } // Latch the first KV-event observation + first-Available time write-once, // the same contract as updateManagedStatus (the gate reads both). if backend.Status.FirstKVEventObservedAt == nil { @@ -254,10 +277,15 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen backend.Status.FirstKVEventObservedAt = at.DeepCopy() } } - // Latch the first-Available time write-once, OR re-anchor it on the - // server-mode→events-only transition (where the prior value is the old - // mode's availability, not the events-only start — see above). - if backend.Status.FirstAvailableAt == nil || transitionedFromServerMode { + // Latch the first-Available time write-once, OR re-anchor it on a + // server-bearing→serverless transition. A typed PodLocal backend is not + // available merely because it has no separately managed workload: wait + // for the connector observation, otherwise a Pod that appears after the + // timeout window would be declared NoKVEventsObserved immediately. + canAnchorAvailability := eventsOnly || !isTypedLMCachePodLocal(backend) || baseStatus == metav1.ConditionTrue + if transitionedFromServerMode && !canAnchorAvailability { + backend.Status.FirstAvailableAt = nil + } else if canAnchorAvailability && (backend.Status.FirstAvailableAt == nil || transitionedFromServerMode) { t := metav1.NewTime(now) backend.Status.FirstAvailableAt = &t } @@ -347,6 +375,8 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend r.probeLimiter.forget(client.ObjectKeyFromObject(backend).String()) return r.patchStatus(ctx, backend, func() { backend.Status.Endpoint = "" + backend.Status.Connector = nil + backend.Status.RemoteStorage = nil // Clear the cache-server-instance latch — cleanupOwnedWorkload // above has just deleted any prior managed Deployment and we // no longer provision one, so a retained UID would advertise @@ -365,6 +395,8 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend // is no longer evaluated for injected engine-pod crash-loops, so clear // any left over from a prior managed state. meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeEngineCompatibility) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeConnectorReady) + meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeRemoteStorageReady) // Reset the KV-event-gate timeout ANCHOR. Unlike firstKVEventObservedAt // (a monotonic observation marker, deliberately kept — see godoc), // firstAvailableAt records when the backend became "up" for the gate's diff --git a/internal/controller/cachebackend_status.go b/internal/controller/cachebackend_status.go index 1f3c9b27..f28802a2 100644 --- a/internal/controller/cachebackend_status.go +++ b/internal/controller/cachebackend_status.go @@ -158,9 +158,30 @@ const ( func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend, endpoint string, dep *appsv1.Deployment, applyOK bool) (time.Duration, error) { now := time.Now() readyStatus, reason, message := managedReadiness(backend, dep) + remoteStatus, remoteReason, remoteMessage := readyStatus, reason, message + if reason == conditionReasonRolloutInProgress { + remoteStatus = metav1.ConditionUnknown + remoteReason = reasonRemoteStoragePending + remoteMessage = "managed remote-storage rollout is still converging" + } else if endpoint == "" { + remoteStatus = metav1.ConditionFalse + remoteReason = reasonRemoteStorageUnavailable + remoteMessage = "managed remote storage has no live Service endpoint" + } else if readyStatus == metav1.ConditionTrue { + remoteReason = reasonRemoteStorageReady + remoteMessage = "managed remote storage workload is Available and its Service endpoint is published" + } else { + remoteReason = reasonRemoteStorageUnavailable + } + if isTypedLMCachePodLocal(backend) { + readyStatus, reason, message = lmCacheMPReadyBase(backend, remoteStatus, remoteReason, remoteMessage) + } // Resolve the stable timeout anchor: the latched FirstAvailableAt, or — the - // first time the workload is Available — now. Using a latched value (not the - // live Deployment Available condition, which resets on a flap) keeps the + // first time the effective backend is Ready — now. For typed PodLocal MP, + // effective readiness is connector health (plus L3 only in fail-closed + // mode), not the optional managed Redis Deployment by itself. Using a + // latched value (not the live Deployment Available condition, which resets + // on a flap) keeps the // firstEventTimeout window monotonic so Degraded stays sticky. anchor := time.Time{} if backend.Status.FirstAvailableAt != nil { @@ -208,6 +229,7 @@ func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backen prevEngineIncompatible := meta.IsStatusConditionFalse(backend.Status.Conditions, conditionTypeEngineCompatibility) err := r.patchStatus(ctx, backend, func() { backend.Status.Endpoint = endpoint + setRemoteStorageStatus(backend, endpoint, remoteStatus, remoteReason, remoteMessage, publishedGen) backend.Status.ObservedGeneration = publishedGen // Latch the first KV-event observation write-once. The poller can later // clear indexParticipation.lastEventAt on a drain, so this durable @@ -906,6 +928,15 @@ type stateSnapshot struct { // rather than re-firing every time a rollout takes an already-event-seen // backend through RolloutInProgress and back to KVEventsObserved. firstEventLatched bool + connector conditionSnapshot + remoteStorage conditionSnapshot +} + +type conditionSnapshot struct { + present bool + status metav1.ConditionStatus + reason string + observedGeneration int64 } // snapshotState captures the prior status values that drive transition events. @@ -918,6 +949,21 @@ func snapshotState(cb *cachev1alpha1.CacheBackend) stateSnapshot { failOpen: statusFailOpen(cb.Status.FailOpen), readyReason: readyConditionReason(cb), firstEventLatched: cb.Status.FirstKVEventObservedAt != nil, + connector: snapshotCondition(cb, conditionTypeConnectorReady), + remoteStorage: snapshotCondition(cb, conditionTypeRemoteStorageReady), + } +} + +func snapshotCondition(cb *cachev1alpha1.CacheBackend, conditionType string) conditionSnapshot { + condition := meta.FindStatusCondition(cb.Status.Conditions, conditionType) + if condition == nil { + return conditionSnapshot{} + } + return conditionSnapshot{ + present: true, + status: condition.Status, + reason: condition.Reason, + observedGeneration: condition.ObservedGeneration, } } @@ -979,6 +1025,8 @@ func (r *CacheBackendReconciler) emitTransitionEvents(cb *cachev1alpha1.CacheBac return } after := snapshotState(cb) + r.emitMPConditionTransition(cb, before.connector, after.connector, conditionTypeConnectorReady) + r.emitMPConditionTransition(cb, before.remoteStorage, after.remoteStorage, conditionTypeRemoteStorageReady) // Generic Conditions[Degraded] transitions. The KV-event gate's Degraded // and Ready flavors carry their own, more specific events below, so @@ -1047,6 +1095,21 @@ func (r *CacheBackendReconciler) emitTransitionEvents(cb *cachev1alpha1.CacheBac } } +func (r *CacheBackendReconciler) emitMPConditionTransition(cb *cachev1alpha1.CacheBackend, before, after conditionSnapshot, conditionType string) { + if !after.present || (before.present && before.status == after.status && before.reason == after.reason && before.observedGeneration == after.observedGeneration) { + return + } + condition := meta.FindStatusCondition(cb.Status.Conditions, conditionType) + if condition == nil { + return + } + eventType := corev1.EventTypeWarning + if condition.Status == metav1.ConditionTrue { + eventType = corev1.EventTypeNormal + } + r.Recorder.Eventf(cb, nil, eventType, condition.Reason, condition.Reason, "%s", condition.Message) +} + // degradedMessage surfaces the Ready=False condition's message (set by // managedReadiness) so the BackendDegraded event names the failure mode // (e.g. "1/3 replicas available") instead of just announcing the transition. diff --git a/internal/enginebinding/metadata.go b/internal/enginebinding/metadata.go index a9993003..6ca2d520 100644 --- a/internal/enginebinding/metadata.go +++ b/internal/enginebinding/metadata.go @@ -10,6 +10,16 @@ import ( ) const ( + // LabelLMCacheMPMetrics marks an engine Pod whose successfully injected + // PodLocal LMCache native sidecar exposes Prometheus metrics. The optional + // observability overlay selects this label and scrapes the sidecar directly; + // it is deliberately not applied to legacy in-process connectors. + LabelLMCacheMPMetrics = "inferencecache.io/lmcache-mp-metrics" + + // LabelLMCacheMPMetricsEnabled is the selector value used by the shipped + // LMCache PodMonitor. + LabelLMCacheMPMetricsEnabled = "true" + // AnnotationSkip lets an operator explicitly opt a pod out of injection. AnnotationSkip = "inferencecache.io/skip-inject" @@ -21,6 +31,12 @@ const ( // admission time and prevents stale name-only binding claims. AnnotationInjectedByUID = "inferencecache.io/injected-by-uid" + // AnnotationInjectedGeneration records the CacheBackend generation whose + // connector/server configuration was rendered into the immutable Pod. It + // lets status distinguish current wiring from Pods that predate a spec + // update and therefore require recreation. + AnnotationInjectedGeneration = "inferencecache.io/injected-generation" + // AnnotationInjectSkipped marks an intentional operator opt-out. AnnotationInjectSkipped = "inferencecache.io/inject-skipped" diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index 3a756a86..0cc95860 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -17,6 +17,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -98,10 +99,12 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { // the registry with a subscriber image — the integration test exists // to gate that end-to-end behaviour. Production operators do the same // by passing --kvevent-subscriber-image to the controller. + registry := newVLLMRegistry(builtinruntime.SubscriberConfig{Image: testSubscriberImage}) + registry.Register(builtinruntime.NewSGLangLMCacheAdapter(builtinruntime.SubscriberConfig{Image: testSubscriberImage})) mgr.GetWebhookServer().Register(WebhookPath, &webhook.Admission{ Handler: &EngineInjector{ Reader: mgr.GetAPIReader(), - Registry: newVLLMRegistry(builtinruntime.SubscriberConfig{Image: testSubscriberImage}), + Registry: registry, }, }) @@ -201,6 +204,9 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { t.Fatalf("annotation %s: got %q want %q (live CacheBackend UID)", AnnotationInjectedByUID, got.Annotations[AnnotationInjectedByUID], string(cb.UID)) } + if got.Annotations[AnnotationInjectedGeneration] != "1" { + t.Fatalf("annotation %s: got %q want %q", AnnotationInjectedGeneration, got.Annotations[AnnotationInjectedGeneration], "1") + } if !containsArgFlag(got.Spec.Containers[0].Args, "--kv-transfer-config") { t.Fatalf("--kv-transfer-config flag not injected; args = %v", got.Spec.Containers[0].Args) } @@ -236,6 +242,7 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { // the apiserver would actually see it. delete(pod2.Annotations, AnnotationInjectedBy) delete(pod2.Annotations, AnnotationInjectedByUID) + delete(pod2.Annotations, AnnotationInjectedGeneration) if err := mgr.GetClient().Create(ctx, pod2); err != nil { t.Fatalf("create second Pod: %v", err) } @@ -277,6 +284,78 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { t.Fatalf("skipped pod unexpectedly has %s env; webhook must not inject engine wiring when %s=true", testEnvLMCacheRemoteURL, AnnotationSkip) } + + // Typed SGLang PodLocal smoke: this goes through a real apiserver so the + // native-sidecar restartPolicy, probes, ports, resource quantities, mounts, + // and SecretKeyRef-capable env shape are schema/defaulting checked rather + // than only compared as Go structs. + typedCB := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "sglang-mp", Namespace: ns}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeLMCache, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + "app": "sglang-mp-test", + }}, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("1Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("2Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }}, + }, + }, + } + if err := mgr.GetClient().Create(ctx, typedCB); err != nil { + t.Fatalf("create typed SGLang CacheBackend: %v", err) + } + typedPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sglang-mp-engine", + Namespace: ns, + Labels: map[string]string{"app": "sglang-mp-test"}, + Annotations: map[string]string{ + "inferencecache.io/lmcache-connector-profile": "sglang-lmcache-mp-v1", + "inferencecache.io/lmcache-client-version": "0.5.3", + }, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "sglang", Image: "sglang:connector-ready", + }}}, + } + if err := mgr.GetClient().Create(ctx, typedPod); err != nil { + t.Fatalf("create typed SGLang Pod: %v", err) + } + var gotTyped corev1.Pod + if err := mgr.GetAPIReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: typedPod.Name}, &gotTyped); err != nil { + t.Fatalf("get typed SGLang Pod: %v", err) + } + var mpServer *corev1.Container + for i := range gotTyped.Spec.InitContainers { + if gotTyped.Spec.InitContainers[i].Name == "lmcache-mp-server" { + mpServer = &gotTyped.Spec.InitContainers[i] + break + } + } + if mpServer == nil { + t.Fatalf("typed native sidecar missing: %+v", gotTyped.Spec.InitContainers) + } + if mpServer.RestartPolicy == nil || *mpServer.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Fatalf("native sidecar restartPolicy = %v, want Always", mpServer.RestartPolicy) + } + if mpServer.StartupProbe == nil || mpServer.ReadinessProbe == nil || mpServer.LivenessProbe == nil { + t.Fatalf("typed native sidecar probes missing: %+v", mpServer) + } + if got := gotTyped.Labels[LabelLMCacheMPMetrics]; got != LabelLMCacheMPMetricsEnabled { + t.Fatalf("typed native sidecar metrics label %s = %q, want %q", + LabelLMCacheMPMetrics, got, LabelLMCacheMPMetricsEnabled) + } } // mustHaveContainerEnv fails the test if the first container's env array diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index 91d6c15b..6ff31f2a 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "sort" + "strconv" "strings" "github.com/go-logr/logr" @@ -69,6 +70,20 @@ const AnnotationInjectedBy = enginebinding.AnnotationInjectedBy // the event. const AnnotationInjectedByUID = enginebinding.AnnotationInjectedByUID +// AnnotationInjectedGeneration records the CacheBackend generation rendered +// into this immutable Pod. The MP status writer uses it to avoid reporting an +// old sidecar configuration as current after the CacheBackend changes. +const AnnotationInjectedGeneration = enginebinding.AnnotationInjectedGeneration + +// LabelLMCacheMPMetrics is stamped only after a typed PodLocal LMCache server +// has been successfully rendered. The optional observability overlay uses it +// to discover the native sidecar's lmcache-http port across engine namespaces. +const LabelLMCacheMPMetrics = enginebinding.LabelLMCacheMPMetrics + +// LabelLMCacheMPMetricsEnabled is the stable selector value for +// [LabelLMCacheMPMetrics]. +const LabelLMCacheMPMetricsEnabled = enginebinding.LabelLMCacheMPMetricsEnabled + // AnnotationInjectSkipped is stamped when the webhook intentionally skips // injection because the operator set [AnnotationSkip]. It lets a persisted pod // distinguish an explicit opt-out from selector drift or fail-open admission. @@ -434,6 +449,13 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi delete(mutated.Annotations, AnnotationInjectSkipped) mutated.Annotations[AnnotationInjectedBy] = cache.Namespace + "/" + cache.Name mutated.Annotations[AnnotationInjectedByUID] = string(cache.UID) + mutated.Annotations[AnnotationInjectedGeneration] = strconv.FormatInt(cache.Generation, 10) + if cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal { + if mutated.Labels == nil { + mutated.Labels = map[string]string{} + } + mutated.Labels[LabelLMCacheMPMetrics] = LabelLMCacheMPMetricsEnabled + } mutatedRaw, err := json.Marshal(mutated) if err != nil { @@ -519,7 +541,8 @@ func (h *EngineInjector) logger(ctx context.Context) logr.Logger { // failOpen builds the admission response for any fail-open return path // AFTER the pod has been decoded. The webhook's contract is that -// AnnotationInjectedBy and AnnotationInjectSkipped on the persisted pod mean +// The injected-by identity/generation annotations and AnnotationInjectSkipped +// on the persisted pod mean // "the webhook successfully made this decision" — those are what the // engine-pod-events controller keys `InjectedByCacheBackend` and // `SkippedByOperator` off of. The annotations are user-controllable (anyone @@ -536,13 +559,15 @@ func (h *EngineInjector) logger(ctx context.Context) logr.Logger { func failOpen(req admission.Request, pod *corev1.Pod, reason string) admission.Response { hasInjectedBy := pod.Annotations[AnnotationInjectedBy] != "" hasInjectedByUID := pod.Annotations[AnnotationInjectedByUID] != "" + hasInjectedGeneration := pod.Annotations[AnnotationInjectedGeneration] != "" hasInjectSkipped := pod.Annotations[AnnotationInjectSkipped] != "" - if !hasInjectedBy && !hasInjectedByUID && !hasInjectSkipped { + if !hasInjectedBy && !hasInjectedByUID && !hasInjectedGeneration && !hasInjectSkipped { return admission.Allowed(reason) } cleared := pod.DeepCopy() delete(cleared.Annotations, AnnotationInjectedBy) delete(cleared.Annotations, AnnotationInjectedByUID) + delete(cleared.Annotations, AnnotationInjectedGeneration) delete(cleared.Annotations, AnnotationInjectSkipped) if len(cleared.Annotations) == 0 { // Avoid emitting an empty-map annotations field; absent is the @@ -568,11 +593,13 @@ func skipInjection(req admission.Request, pod *corev1.Pod) admission.Response { } delete(mutated.Annotations, AnnotationInjectedBy) delete(mutated.Annotations, AnnotationInjectedByUID) + delete(mutated.Annotations, AnnotationInjectedGeneration) mutated.Annotations[AnnotationInjectSkipped] = InjectSkippedReasonSkipAnnotation if pod.Annotations[AnnotationInjectSkipped] == InjectSkippedReasonSkipAnnotation && pod.Annotations[AnnotationInjectedBy] == "" && - pod.Annotations[AnnotationInjectedByUID] == "" { + pod.Annotations[AnnotationInjectedByUID] == "" && + pod.Annotations[AnnotationInjectedGeneration] == "" { return admission.Allowed("skipped via " + AnnotationSkip) } raw, err := json.Marshal(mutated) @@ -648,6 +675,9 @@ func effectiveEndpoint(cache *cachev1alpha1.CacheBackend) string { } return ep } + if cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" && cache.Status.RemoteStorage != nil { + return strings.TrimSpace(cache.Status.RemoteStorage.Endpoint) + } return strings.TrimSpace(cache.Status.Endpoint) } diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index cd49931d..b2b14332 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -315,6 +315,9 @@ func TestHandle_MatchAndInject(t *testing.T) { if got, want := mutated.Annotations[AnnotationInjectedByUID], string(cb.UID); got != want { t.Fatalf("annotation %s: got %q, want %q (matched CR UID)", AnnotationInjectedByUID, got, want) } + if got, want := mutated.Annotations[AnnotationInjectedGeneration], fmt.Sprint(cb.Generation); got != want { + t.Fatalf("annotation %s: got %q, want %q", AnnotationInjectedGeneration, got, want) + } mustHaveArgPair(t, mutated, "--model", "Qwen/Qwen2.5-0.5B-Instruct") mustHaveArgFlag(t, mutated, "--kv-transfer-config") } @@ -408,6 +411,71 @@ func TestHandle_MatchAndInject_SGLang(t *testing.T) { if !hasWorker { t.Fatalf("MP-worker sidecar not injected; initContainers = %+v", mutated.Spec.InitContainers) } + if got := mutated.Labels[LabelLMCacheMPMetrics]; got != "" { + t.Fatalf("legacy topology-less SGLang pod got typed MP metrics label %s=%q", LabelLMCacheMPMetrics, got) + } +} + +func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { + const ns = "engines" + cb := readyCacheBackend("sg-typed", ns, map[string]string{"app": "sglang"}) + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + chunkSize := int32(256) + cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + ChunkSizeTokens: &chunkSize, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + }, + }}, + } + h := newHandler(t, cb) + pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[adapterruntime.AnnotationLMCacheConnectorProfile] = "sglang-lmcache-mp-v1" + pod.Annotations[adapterruntime.AnnotationLMCacheClientVersion] = "0.5.3" + req := newRequest(t, pod, ns) + + resp := h.Handle(context.Background(), req) + if !resp.Allowed || len(resp.Patches) == 0 { + t.Fatalf("typed SGLang injection: Allowed=%v patches=%d result=%+v", resp.Allowed, len(resp.Patches), resp.Result) + } + mutated := applyPatches(t, req.Object.Raw, resp) + mustHaveArgFlag(t, mutated, "--enable-lmcache") + mustHaveArgPair(t, mutated, "--lmcache-config-file", "/var/run/inference-cache/lmcache/client.yaml") + server := findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") + if server == nil { + t.Fatalf("typed MP server missing: %+v", mutated.Spec.InitContainers) + } + if server.Image != cb.Spec.LMCache.PodLocal.Server.Image || server.Image == mutated.Spec.Containers[0].Image { + t.Fatalf("server image = %q, engine image = %q", server.Image, mutated.Spec.Containers[0].Image) + } + joined := strings.Join(append(server.Command, server.Args...), " ") + if !strings.Contains(joined, "lmcache server") || !strings.Contains(joined, "--http-port 8080") || strings.Contains(joined, "python3 -m") { + t.Fatalf("typed server command = %s", joined) + } + if server.StartupProbe == nil || server.ReadinessProbe == nil || server.LivenessProbe == nil { + t.Fatalf("typed server probes missing: %+v", server) + } + if got := mutated.Labels[LabelLMCacheMPMetrics]; got != LabelLMCacheMPMetricsEnabled { + t.Fatalf("label %s = %q, want %q", LabelLMCacheMPMetrics, got, LabelLMCacheMPMetricsEnabled) + } + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-worker") != nil { + t.Fatalf("typed wire fell through to legacy worker: %+v", mutated.Spec.InitContainers) + } } func TestHandle_MatchAndInject_SGLangHiCacheWithoutEndpoint(t *testing.T) { @@ -1050,8 +1118,9 @@ func TestHandle_EventsOnly_NoSubscriber_StripsForgedInjectedBy(t *testing.T) { h := newHandler(t, cb) // no subscriber image → nothing to wire pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) pod.Annotations = map[string]string{ - AnnotationInjectedBy: ns + "/routing-only", - AnnotationInjectedByUID: string(cb.UID), + AnnotationInjectedBy: ns + "/routing-only", + AnnotationInjectedByUID: string(cb.UID), + AnnotationInjectedGeneration: fmt.Sprint(cb.Generation), } req := newRequest(t, pod, ns) @@ -1066,6 +1135,9 @@ func TestHandle_EventsOnly_NoSubscriber_StripsForgedInjectedBy(t *testing.T) { if got := mutated.Annotations[AnnotationInjectedByUID]; got != "" { t.Fatalf("forged %s must be stripped on no-wiring fail-open; got %q", AnnotationInjectedByUID, got) } + if got := mutated.Annotations[AnnotationInjectedGeneration]; got != "" { + t.Fatalf("forged %s must be stripped on no-wiring fail-open; got %q", AnnotationInjectedGeneration, got) + } } func TestHandle_EventsOnly_PrebakedSubscriber_NotClaimedNoStamp(t *testing.T) { @@ -1726,6 +1798,15 @@ func findContainer(pod *corev1.Pod, name string) *corev1.Container { return nil } +func findInitContainerByName(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + func containerNames(pod *corev1.Pod) []string { out := make([]string, len(pod.Spec.Containers)) for i, c := range pod.Spec.Containers { @@ -2143,9 +2224,10 @@ func TestHandle_SkipAnnotationStampsSkippedReasonAndClearsInjectedBy(t *testing. h := newHandler(t, cb) pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) pod.Annotations = map[string]string{ - AnnotationSkip: "true", - AnnotationInjectedBy: ns + "/" + cb.Name, - AnnotationInjectedByUID: string(cb.UID), + AnnotationSkip: "true", + AnnotationInjectedBy: ns + "/" + cb.Name, + AnnotationInjectedByUID: string(cb.UID), + AnnotationInjectedGeneration: fmt.Sprint(cb.Generation), } req := newRequest(t, pod, ns) @@ -2163,6 +2245,9 @@ func TestHandle_SkipAnnotationStampsSkippedReasonAndClearsInjectedBy(t *testing. if got := mutated.Annotations[AnnotationInjectedByUID]; got != "" { t.Fatalf("annotation %s = %q, want cleared on skip path", AnnotationInjectedByUID, got) } + if got := mutated.Annotations[AnnotationInjectedGeneration]; got != "" { + t.Fatalf("annotation %s = %q, want cleared on skip path", AnnotationInjectedGeneration, got) + } } func mustHaveEnv(t *testing.T, pod *corev1.Pod, name, value string) { diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index 8e531171..fa6e651c 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -12,10 +12,14 @@ import ( cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + kvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" ) -const lmcacheKVEventPort int32 = 5557 +const ( + lmcacheKVEventPort int32 = 5557 + lmcacheMPHTTPPort int32 = 8080 +) var sha256ImagePattern = regexp.MustCompile(`^[^[:space:]@]+@sha256:[a-f0-9]{64}$`) @@ -182,6 +186,9 @@ func validateMPServer( } else if port == lmcacheKVEventPort { errs = append(errs, field.Invalid(path.Child("port"), port, fmt.Sprintf("collides with the engine KV-event publisher port %d", lmcacheKVEventPort))) + } else if port == lmcacheMPHTTPPort { + errs = append(errs, field.Invalid(path.Child("port"), port, + fmt.Sprintf("collides with the LMCache MP HTTP health/control port %d", lmcacheMPHTTPPort))) } if l1Capacity == nil || l1Capacity.Sign() <= 0 { @@ -286,9 +293,11 @@ func validateMPServerResourceRequirements(resources corev1.ResourceRequirements, return errs } -// rejectUnimplementedRedisBindingFeatures keeps the newly typed credential/TLS -// contract from becoming accepted-but-ignored configuration. Phase 2 removes -// this gate when the structured runtime binding and secret mounts are rendered. +// rejectUnimplementedRedisBindingFeatures permits the Phase-2 SGLang MP auth +// path while keeping every unsupported LMCache 0.5.3 RESP feature explicit. +// That adapter supports username/password, but not TLS or logical database +// selection. Managed Redis currently provisions the default user, so its +// password may be configured but an ACL username may not. func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) field.ErrorList { if cb == nil || cb.Spec.RemoteStorage == nil || cb.Spec.RemoteStorage.Redis == nil { return nil @@ -297,16 +306,51 @@ func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) fie path := field.NewPath("spec", "remoteStorage", "redis") var errs field.ErrorList if redis.Authentication != nil { - errs = append(errs, field.Forbidden(path.Child("authentication"), - "Redis authentication is typed but not rendered until Phase 2; refusing inert credentials")) + authPath := path.Child("authentication") + isSGLangMP := cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeSGLang && + cb.Spec.LMCache != nil && cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal + if !isSGLangMP { + errs = append(errs, field.Forbidden(authPath, + "Redis authentication is currently rendered only by the SGLang PodLocal LMCache MP adapter")) + } else { + if redis.Authentication.Username != nil { + errs = append(errs, validateRedisSecretKeySelector(*redis.Authentication.Username, authPath.Child("username"))...) + } + errs = append(errs, validateRedisSecretKeySelector(redis.Authentication.Password, authPath.Child("password"))...) + if cb.Spec.RemoteStorage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged && redis.Authentication.Username != nil { + errs = append(errs, field.Forbidden(authPath.Child("username"), + "managed Redis provisions the default user and currently supports password authentication only")) + } + } } if redis.TLS != nil { errs = append(errs, field.Forbidden(path.Child("tls"), - "Redis TLS is typed but not rendered until Phase 2; refusing inert TLS configuration")) + "the pinned LMCache 0.5.3 resp adapter does not support TLS; refusing inert TLS configuration")) } if redis.Database != nil { errs = append(errs, field.Forbidden(path.Child("database"), - "Redis database selection is typed but not rendered until Phase 2; refusing inert adapter configuration")) + "the pinned LMCache 0.5.3 resp adapter does not support database selection; refusing inert adapter configuration")) + } + return errs +} + +func validateRedisSecretKeySelector(selector corev1.SecretKeySelector, path *field.Path) field.ErrorList { + var errs field.ErrorList + name := strings.TrimSpace(selector.Name) + if name == "" { + errs = append(errs, field.Required(path.Child("name"), "a namespace-local Secret name is required")) + } else if messages := kvalidation.IsDNS1123Subdomain(name); len(messages) > 0 { + errs = append(errs, field.Invalid(path.Child("name"), selector.Name, strings.Join(messages, "; "))) + } + key := strings.TrimSpace(selector.Key) + if key == "" { + errs = append(errs, field.Required(path.Child("key"), "a Secret data key is required")) + } else if messages := kvalidation.IsConfigMapKey(key); len(messages) > 0 { + errs = append(errs, field.Invalid(path.Child("key"), selector.Key, strings.Join(messages, "; "))) + } + if selector.Optional != nil && *selector.Optional { + errs = append(errs, field.Forbidden(path.Child("optional"), + "authentication Secret keys are required; optional credentials could start the cache plane unauthenticated or unusable")) } return errs } diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index a9cfb5e6..8d938871 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -210,6 +210,13 @@ func TestValidateLMCacheTopology(t *testing.T) { }, wantField: "spec.lmCache.podLocal.server.port", }, + { + name: "HTTP health port collision", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Port = lmcacheMPHTTPPort + }, + wantField: "spec.lmCache.podLocal.server.port", + }, { name: "memory request has no headroom", mutate: func(cb *cachev1alpha1.CacheBackend) { @@ -282,10 +289,99 @@ func TestRejectUnimplementedRedisBindingFeatures(t *testing.T) { t.Fatalf("errors = %v, want database rejection", errs) } _, err := shippingValidator().ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), "not rendered until Phase 2") { - t.Fatalf("ValidateCreate error = %v, want inert-binding rejection", err) + if err == nil || !strings.Contains(err.Error(), "does not support database selection") { + t.Fatalf("ValidateCreate error = %v, want pinned-adapter capability rejection", err) } if strings.Contains(err.Error(), "provider workload configuration") { t.Fatalf("external Redis connection settings were misclassified as managed workload config: %v", err) } } + +func TestValidateRedisAuthenticationForSGLangPodLocal(t *testing.T) { + newAuthBackend := func(ownership cachev1alpha1.CacheBackendRemoteStorageOwnership) *cachev1alpha1.CacheBackend { + cb := validPodLocalMPBackend() + cb.Name = "mp" + cb.Namespace = "default" + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: ownership, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{Authentication: &cachev1alpha1.RedisAuthenticationSpec{ + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, + Key: "password", + }, + }}, + } + if ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { + cb.Spec.RemoteStorage.Endpoint = "redis.example:6379" + } else { + cb.Spec.RemoteStorage.Redis.Image = "redis:7.4-alpine" + } + return cb + } + + for _, ownership := range []cachev1alpha1.CacheBackendRemoteStorageOwnership{ + cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + } { + t.Run(string(ownership)+" password", func(t *testing.T) { + cb := newAuthBackend(ownership) + if errs := rejectUnimplementedRedisBindingFeatures(cb); len(errs) != 0 { + t.Fatalf("authentication errors = %v", errs) + } + if _, err := shippingValidator().ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("ValidateCreate: %v", err) + } + }) + } + + t.Run("external ACL username", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + cb.Spec.RemoteStorage.Redis.Authentication.Username = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "username", + } + if errs := rejectUnimplementedRedisBindingFeatures(cb); len(errs) != 0 { + t.Fatalf("external ACL authentication errors = %v", errs) + } + }) + + t.Run("managed ACL username rejected", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged) + cb.Spec.RemoteStorage.Redis.Authentication.Username = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, Key: "username", + } + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 1 || !strings.Contains(errs[0].Field, "username") { + t.Fatalf("errors = %v, want managed username rejection", errs) + } + }) + + t.Run("optional password rejected", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + optional := true + cb.Spec.RemoteStorage.Redis.Authentication.Password.Optional = &optional + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 1 || !strings.Contains(errs[0].Field, "optional") { + t.Fatalf("errors = %v, want optional rejection", errs) + } + }) + + t.Run("empty selector rejected", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + cb.Spec.RemoteStorage.Redis.Authentication.Password = corev1.SecretKeySelector{} + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 2 { + t.Fatalf("errors = %v, want name and key rejections", errs) + } + }) + + t.Run("vLLM remains rejected until its MP adapter lands", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "SGLang PodLocal") { + t.Fatalf("errors = %v, want runtime-scoped rejection", errs) + } + }) +} diff --git a/site/content/en/docs/administration/_index.md b/site/content/en/docs/administration/_index.md index 841e221f..8db78b4e 100644 --- a/site/content/en/docs/administration/_index.md +++ b/site/content/en/docs/administration/_index.md @@ -14,7 +14,7 @@ This section covers the operational concerns of running inference-cache in produ ### [Observability & Alerts]({{< relref "/docs/administration/observability-and-alerts/" >}}) The opt-in Prometheus alert bundle, what each alert means and when it fires, and the -scraping you must configure for both the server and the controller pod. +scraping you must configure for the server, controller, and injected LMCache MP sidecars. ### [Index sizing]({{< relref "/docs/administration/index-sizing/" >}}) diff --git a/site/content/en/docs/administration/observability-and-alerts.md b/site/content/en/docs/administration/observability-and-alerts.md index 8176a7fd..2f3c57fd 100644 --- a/site/content/en/docs/administration/observability-and-alerts.md +++ b/site/content/en/docs/administration/observability-and-alerts.md @@ -17,18 +17,20 @@ unknown `apiVersion`. kubectl apply -k config/observability ``` -This ships three resources in the `inference-cache-system` namespace: +This ships four resources in the `inference-cache-system` namespace: - a **`ServiceMonitor`** — scrapes `inference-cache-server:8080/metrics`; - a **`PodMonitor`** — scrapes the controller pod's `:8080/metrics` (required for the controller-side alerts to have a series to evaluate — the controller has no Service in front of it); +- a second **`PodMonitor`** — discovers successfully injected PodLocal LMCache native + sidecars across workload namespaces and scrapes `lmcache-http:8080/metrics`; - a **`PrometheusRule`** — the alerts. `make verify-prometheus` lints and unit-tests the rules. {{% alert title="Selector mismatch fails silently" color="warning" %}} -All three custom resources carry example labels (`prometheus: k8s`) matching the upstream +All four custom resources carry example labels (`prometheus: k8s`) matching the upstream kube-prometheus stack. The `kube-prometheus-stack` Helm chart uses a *different* convention (`release: `, no `prometheus:` label). If your `Prometheus` custom resource's `ruleSelector` / `serviceMonitorSelector` / `podMonitorSelector` uses a different label set, @@ -37,8 +39,8 @@ selectors with `kubectl get prometheus -A -o yaml` and relabel the CRs to match. {{% /alert %}} For vanilla Prometheus (ConfigMap mounts, `prometheus.serverFiles`), use the flat -`config/observability/alerting-rules.yaml` and configure scraping yourself — **for both the -server and the controller pod**. Scope per install with `kubernetes_sd_configs` + a +`config/observability/alerting-rules.yaml` and configure scraping yourself — **for the +server, controller pod, and injected PodLocal LMCache sidecars**. Scope per install with `kubernetes_sd_configs` + a `relabel_configs` that copies `__meta_kubernetes_namespace` to `namespace` (the alerts scope per install by that label). @@ -60,7 +62,7 @@ traffic/rate/eviction thresholds). {{% alert title="LMCacheT2NoHits needs an extra scrape" color="warning" %}} `LMCacheT2NoHits` reads `vllm:external_prefix_cache_*` from the **engine pods directly**, not -from inference-cache. The shipped `ServiceMonitor` covers only `inference-cache-server`. To +from inference-cache. The shipped scrape configs do not collect vLLM's own metrics. To make this alert effective, add a separate `PodMonitor` for your engine Deployment (scoped to the pods your `CacheBackend.spec.engineSelector` matches) that preserves both `namespace` and `pod` labels. diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index fdc36e0d..392d567a 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -97,12 +97,12 @@ For prometheus-operator / kube-prometheus installs: kubectl apply -k config/observability ``` -This ships three resources: a `ServiceMonitor` (scrapes `inference-cache-server:8080`), a -`PodMonitor` (scrapes the controller pod's `:8080` — required for the controller-side -alerts to have a series to evaluate), and a `PrometheusRule` carrying the alerts. +This ships four resources: a `ServiceMonitor` for `inference-cache-server:8080`, a +`PodMonitor` for the controller pod's `:8080`, a cross-namespace `PodMonitor` for +successfully injected PodLocal LMCache sidecars, and a `PrometheusRule` carrying the alerts. {{% alert title="Caveat — Prometheus Operator selectors" color="warning" %}} -All three custom resources carry example labels (`prometheus: k8s`) that match the upstream +All four custom resources carry example labels (`prometheus: k8s`) that match the upstream kube-prometheus stack. If your `Prometheus` custom resource's selectors use a different label set (for example `release: my-prom` from the `kube-prometheus-stack` Helm chart), `kubectl apply -k` succeeds but Prometheus silently ignores the resources. See diff --git a/site/content/en/docs/reference/metrics.md b/site/content/en/docs/reference/metrics.md index c4b3c0ce..bacc9d97 100644 --- a/site/content/en/docs/reference/metrics.md +++ b/site/content/en/docs/reference/metrics.md @@ -6,10 +6,12 @@ description: > Every inferencecache_* series, its labels, and which binary emits it. --- -Both binaries expose Prometheus metrics on their pod's `:8080/metrics`, all prefixed +Both inference-cache binaries expose Prometheus metrics on their pod's `:8080/metrics`, all prefixed `inferencecache_*`. The two binaries use **separate registries** — the server's series cover the index and gRPC handlers; the controller's cover the reconcilers. Standard `go_*` and -`process_*` collectors are also present but are not part of this schema. +`process_*` collectors are also present but are not part of this schema. Successfully +injected PodLocal LMCache sidecars expose their upstream `lmcache_mp_*` metrics separately; +the observability overlay includes a cross-namespace PodMonitor for them. ## Server metrics (`inference-cache-server`) @@ -65,6 +67,7 @@ alerts have no series to evaluate. | Server | `:8080` (`--http-bind-address`) | `/healthz`, `/readyz` (unauth), `/metrics` | | Server | `:8081` (`--snapshot-bind-address`) | `/snapshot`, `/policy`, `/probe` (auth-gated) | | Controller | `:8080` (`--metrics-bind-address`) | `/metrics` (unauth by default; `--metrics-secure` to gate) | +| Injected LMCache MP sidecar | `:8080` (`lmcache-http`) | `/healthcheck`, `/metrics` (unauthenticated on the Pod network) | ## Related pages From 1d0a233730f38de0629ef15bba7d7df69ea84f44 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Sun, 9 Aug 2026 22:33:27 -0700 Subject: [PATCH 03/13] Start SGLang PodLocal MP production baseline Signed-off-by: Yue Sun --- config/samples/README.md | 8 +- ...ackend-sglang-podlocal-external-redis.yaml | 48 ++++++++ ...achebackend-sglang-podlocal-host-only.yaml | 38 +++++++ ...backend-sglang-podlocal-managed-redis.yaml | 48 ++++++++ .../lmcache-multiprocess-migration-roadmap.md | 68 ++++++++--- .../scripts/default_install_smoke.sh | 107 ++++++++++++++++++ .../builtin/runtime/sglang_lmcache.go | 51 ++++++++- .../builtin/runtime/sglang_lmcache_test.go | 41 ++++++- .../webhook/pod/envtest_integration_test.go | 2 +- internal/webhook/pod/podinjector_test.go | 21 ++++ test/fixtures/sglang-lmcache/Dockerfile | 17 +++ 11 files changed, 427 insertions(+), 22 deletions(-) create mode 100644 config/samples/cachebackend-sglang-podlocal-external-redis.yaml create mode 100644 config/samples/cachebackend-sglang-podlocal-host-only.yaml create mode 100644 config/samples/cachebackend-sglang-podlocal-managed-redis.yaml create mode 100644 test/fixtures/sglang-lmcache/Dockerfile diff --git a/config/samples/README.md b/config/samples/README.md index 350dae5a..eb45b51b 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -15,8 +15,12 @@ multi-tenant, Namespaces): - **`cachebackend-*.yaml`** — focused hand-curated canonical CacheBackend examples, including the [`cachebackend-sglang-hicache.yaml`](cachebackend-sglang-hicache.yaml) - engine-local example. The `recipe-*.yaml` catalog is the maintained entry - point for LMCache scenarios. + engine-local example and the typed SGLang PodLocal LMCache examples for + [host-only](cachebackend-sglang-podlocal-host-only.yaml), + [managed Redis](cachebackend-sglang-podlocal-managed-redis.yaml), and + [external Redis](cachebackend-sglang-podlocal-external-redis.yaml). The + `recipe-*.yaml` catalog remains the maintained entry point for legacy + in-process LMCache scenarios until the repository-wide Phase-5 migration. ## Recipe catalog diff --git a/config/samples/cachebackend-sglang-podlocal-external-redis.yaml b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml new file mode 100644 index 00000000..080c9417 --- /dev/null +++ b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed SGLang PodLocal LMCache MP with an externally managed Redis remote tier. +# Create Secret/redis-auth with key password in this namespace before starting +# matching engines. LMCache 0.5.3 RESP supports authentication but not TLS, so +# keep this endpoint on a trusted private network; admission rejects an inert +# remoteStorage.redis.tls block instead of silently sending plaintext. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-podlocal-external-redis +spec: + runtime: SGLang + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: sglang-mp-external-redis + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + remoteStorage: + provider: Redis + ownership: External + endpoint: redis.example.internal:6379 + redis: + authentication: + password: + name: redis-auth + key: password + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-sglang-podlocal-host-only.yaml b/config/samples/cachebackend-sglang-podlocal-host-only.yaml new file mode 100644 index 00000000..88b31e6b --- /dev/null +++ b/config/samples/cachebackend-sglang-podlocal-host-only.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed SGLang PodLocal LMCache MP with no remote tier. The inference-owner Pod +# image must contain lmcache==0.5.3 and declare the connector capability shown +# in docs/design/lmcache-multiprocess-migration-roadmap.md. Its launch args must +# explicitly set --page-size to a divisor of chunkSizeTokens. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-podlocal-host-only +spec: + runtime: SGLang + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: sglang-mp + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml new file mode 100644 index 00000000..4cbfaf4e --- /dev/null +++ b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed SGLang PodLocal LMCache MP with a controller-managed, development Redis +# remote tier. Redis is ephemeral soft state and intentionally has one replica. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-podlocal-managed-redis +spec: + runtime: SGLang + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: sglang-mp-managed-redis + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + remoteStorage: + provider: Redis + ownership: Managed + redis: + image: redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 + resources: + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: "2" + memory: 8Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 069126ba..518e4051 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -356,7 +356,7 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 0 | Design freeze, consumer audit, version/Kubernetes baseline | none | complete | | 1 | MP-only API and admission/status contracts | Phase 0 | complete | | 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | complete | -| 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | not started | +| 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | in progress — non-GPU baseline complete; GPU matrix pending | | 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | not started | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | | 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 finding | @@ -667,17 +667,56 @@ The exact LMCache 0.5.3 source constrains these runtime capability boundaries: Use the already working SGLang path to validate the common MP server under parallelism and failure before adding vLLM. -### Functional scope - -- [ ] SGLang + PodLocal + no remote L3. -- [ ] SGLang + PodLocal + managed Redis development profile. -- [ ] SGLang + PodLocal + external Redis production profile. -- [ ] ReadWrite role; reject unsupported role splits. -- [ ] Pinned SGLang/LMCache/CUDA image tuple. +Current state: the non-GPU control-plane and connector compatibility baseline +is complete. No Phase 3 GPU data-path or runtime-failure result is claimed yet. + +### Completed non-GPU validation + +- [x] Add typed host-only, managed-Redis, and external-Redis SGLang PodLocal + samples; all three pass CRD defaulting and admission. +- [x] Add the connector-ready SGLang fixture Dockerfile under + `test/fixtures/sglang-lmcache`, based on the pinned SGLang digest with + exactly `lmcache==0.5.3`. +- [x] Build the fixture locally for linux/amd64 and verify SGLang + `0.5.13.post1`, CUDA 13.0.1, LMCache 0.5.3, `LMCacheMPConnector` import, + and CLI parsing for LMCache, explicit page size, and TP=2 flags. This is + compatibility preflight only; the image has not run inference. +- [x] Require an explicit SGLang `--page-size` and reject missing, malformed, + duplicate, non-positive, and declared chunk-incompatible values before + rendering the MP wire. LMCache retains its authoritative runtime check + against the effective page size. +- [x] Verify the Pod webhook renders the common MP server atomically and admits + an incompatible engine unchanged with an actionable fail-open diagnostic. +- [x] Persist a typed SGLang Pod through an envtest kube-apiserver/etcd and + verify the native-sidecar schema/defaulting surface. +- [x] Install the controller and webhooks in a Kubernetes 1.32 kind cluster, + create a matching SGLang Pod through the live mutating webhook, read the + persisted injected Pod back from the API server, and verify image, + restart policy, probes, resources, MP arguments, engine arguments, + annotations, labels, and shared mounts. The Pod was deliberately left + unscheduled, so no engine or sidecar container ran. +- [x] Pass the repository regression gates: full Go tests, focused envtest, + sample verification (24 pass, 2 intentional skips), default-install + smoke, Go vet, Prometheus rules, docs sync, REUSE, and DCO. + +### GPU/runtime functional scope + +- [ ] SGLang + PodLocal + no remote L3. Typed sample and admission wire are + complete; GPU KV execution is pending. +- [ ] SGLang + PodLocal + managed Redis development profile. Typed sample, + managed workload rendering, and admission pass; GPU KV execution is + pending. +- [ ] SGLang + PodLocal + external Redis production profile. Typed sample and + credential binding admission pass; GPU KV execution is pending. +- [x] ReadWrite role; reject unsupported role splits. +- [ ] Pinned SGLang/LMCache/CUDA image tuple. Local build and compatibility + preflight pass; registry digest and GPU execution are pending. ### Correctness work -- [ ] Validate LMCache chunk size against the effective SGLang page size. +- [ ] Exercise LMCache's runtime chunk-size check against the effective SGLang + page size on the pinned GPU tuple. The explicit-value admission guard and + its edge-case tests are complete. - [ ] Validate TP=1 and TP=2 at minimum. - [ ] Prove store → engine-GPU flush → retrieve from MP L1. - [ ] Prove cross-Pod store/retrieve through Redis with fresh engine and MP L1. @@ -697,12 +736,15 @@ parallelism and failure before adding vLLM. ### Operability work -- [ ] `ConnectorReady` reflects MP server health. -- [ ] `RemoteStorageReady` reflects Redis independently. -- [ ] Metrics prove lookup/store/retrieve/hit behavior. +- [ ] `ConnectorReady` reflects MP server health. Condition-transition tests + pass; live SGLang failure evidence is pending. +- [ ] `RemoteStorageReady` reflects Redis independently. Condition-transition + tests pass; live Redis failure evidence is pending. +- [ ] Metrics prove lookup/store/retrieve/hit behavior. Metrics discovery and + Pod labeling pass; real KV traffic evidence is pending. - [ ] Logs identify engine Pod, backend, model, MP instance, and L3 adapter without exposing credentials. -- [ ] Default-install smoke creates a matching SGLang engine Pod through the +- [x] Default-install smoke creates a matching SGLang engine Pod through the live webhook and inspects the actual injected wire. ### Exit criteria diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index f88d791a..e92eb6a4 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -77,6 +77,11 @@ # Deployment/Service/HPA and publishes no endpoint, while the committed # SGLang+Managed-Redis sample explicitly creates a redis-l2 Deployment + # Service and publishes its RESP endpoint. No engine traffic is required. +# 8d. Typed SGLang PodLocal admission: a matching, connector-declared SGLang +# Pod is actually persisted through the installed mutating webhook while +# pinned to an impossible node selector. The smoke reads the persisted Pod +# back and asserts the common lmcache-mp-server native sidecar, probes, +# resources, shared mounts, engine flags/env, and injection identity. # 9. Canonical External ownership end-to-end: applying the committed # config/samples/cachebackend-external.yaml drives the CacheBackend # mutating webhook default (spec.replicas=1), renders NO @@ -369,6 +374,8 @@ EXT_SMOKE_POD_NAME="${EXT_SMOKE_POD_NAME:-smoke-engine}" CANONICAL_SMOKE_NS="${CANONICAL_SMOKE_NS:-ic-smoke-canonical-cache}" CANONICAL_HOST_ONLY_CB="cachebackend-sglang-host-only" CANONICAL_REDIS_CB="cachebackend-sglang" +CANONICAL_TYPED_CB="sglang-podlocal-host-only" +CANONICAL_TYPED_POD="sglang-podlocal-admission" # Events-only-backend smoke fixture identifiers. Declared up front so the # diagnostics helper can reference them even if the smoke aborts before the @@ -447,6 +454,8 @@ collect_diagnostics() { >"$LOG_DIR/canonical-cachebackends.yaml" 2>&1 || true kubectl -n "$CANONICAL_SMOKE_NS" get deploy,svc,hpa -o yaml \ >"$LOG_DIR/canonical-provider-workloads.yaml" 2>&1 || true + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml \ + >"$LOG_DIR/sglang-podlocal-admission.yaml" 2>&1 || true # External-backend smoke artefacts. Best-effort — the CR/pod may not # exist if the smoke aborted before that section. kubectl get cb -A -o wide \ @@ -2246,6 +2255,104 @@ if [ "$redis_provider" != "Redis" ] || [ "$redis_ownership" != "Managed" ] || \ fi log "canonical Managed Redis hierarchy rendered redis-l2 and endpoint=$redis_endpoint" +# Create (not only server-side dry-run) a matching SGLang Pod through the live +# webhook and inspect the object persisted by the apiserver. The impossible node +# selector keeps kubelet from pulling either large GPU image; admission still +# executes the complete mutation and Kubernetes schema/defaulting path. +kubectl -n "$CANONICAL_SMOKE_NS" apply \ + -f config/samples/cachebackend-sglang-podlocal-host-only.yaml >/dev/null \ + || fail "typed SGLang PodLocal CacheBackend sample failed to apply" + +typed_sglang_pod="$(mktemp "$tmpdir/sglang-podlocal-admission.XXXXXX.yaml")" +cat >"$typed_sglang_pod" <<'EOF' +apiVersion: v1 +kind: Pod +metadata: + name: sglang-podlocal-admission + labels: + inferencecache.io/runtime: sglang-mp + annotations: + inferencecache.io/lmcache-connector-profile: sglang-lmcache-mp-v1 + inferencecache.io/lmcache-client-version: "0.5.3" +spec: + nodeSelector: + inferencecache.io/install-smoke-never-schedule: "true" + containers: + - name: sglang + image: example.invalid/sglang-lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + command: ["python3", "-m", "sglang.launch_server"] + args: + - --model-path=meta-llama/Meta-Llama-3-8B-Instruct + - --page-size=64 + - --tensor-parallel-size=1 +EOF +kubectl -n "$CANONICAL_SMOKE_NS" create -f "$typed_sglang_pod" >/dev/null \ + || fail "matching typed SGLang Pod did not pass the live mutating webhook" + +typed_injected_by="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true)" +typed_metrics_label="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.metadata.labels.inferencecache\.io/lmcache-mp-metrics}' 2>/dev/null || true)" +typed_server_image="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].image}' 2>/dev/null || true)" +typed_server_restart="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].restartPolicy}' 2>/dev/null || true)" +typed_server_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].args}' 2>/dev/null || true)" +typed_server_memory_request="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].resources.requests.memory}' 2>/dev/null || true)" +typed_server_memory_limit="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].resources.limits.memory}' 2>/dev/null || true)" +typed_server_probe_path="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].readinessProbe.httpGet.path}' 2>/dev/null || true)" +typed_engine_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="sglang")].args}' 2>/dev/null || true)" +typed_engine_experimental="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="sglang")].env[?(@.name=="LMCACHE_USE_EXPERIMENTAL")].value}' 2>/dev/null || true)" +typed_engine_mounts="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="sglang")].volumeMounts[*].mountPath}' 2>/dev/null || true)" + +expected_mp_image="lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13" +if [ "$typed_injected_by" != "$CANONICAL_SMOKE_NS/$CANONICAL_TYPED_CB" ] || \ + [ "$typed_metrics_label" != "true" ] || \ + [ "$typed_server_image" != "$expected_mp_image" ] || \ + [ "$typed_server_restart" != "Always" ] || \ + [ "$typed_server_memory_request" != "5Gi" ] || \ + [ "$typed_server_memory_limit" != "6Gi" ] || \ + [ "$typed_server_probe_path" != "/healthcheck" ] || \ + [ "$typed_engine_experimental" != "True" ]; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true + fail "typed SGLang persisted wire metadata is wrong: injectedBy=$typed_injected_by metrics=$typed_metrics_label image=$typed_server_image restart=$typed_server_restart memory=$typed_server_memory_request/$typed_server_memory_limit probe=$typed_server_probe_path experimental=$typed_engine_experimental" +fi +if ! jq -e ' + (index("--port") as $port | $port != null and .[$port + 1] == "5555") and + (index("--chunk-size") as $chunk | $chunk != null and .[$chunk + 1] == "256") and + index("--l2-adapter") == null + ' >/dev/null <<<"$typed_server_args"; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true + fail "typed host-only SGLang MP server args are wrong: $typed_server_args" +fi +if ! jq -e ' + index("--page-size=64") != null and + index("--enable-lmcache") != null and + (index("--lmcache-config-file") as $config | $config != null and + .[$config + 1] == "/var/run/inference-cache/lmcache/client.yaml") + ' >/dev/null <<<"$typed_engine_args"; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true + fail "typed SGLang engine args are incomplete: $typed_engine_args" +fi +case " $typed_engine_mounts " in + *" /dev/shm "*) : ;; + *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true + fail "typed SGLang engine /dev/shm mount is missing: $typed_engine_mounts" ;; +esac +case " $typed_engine_mounts " in + *" /var/run/inference-cache/lmcache "*) : ;; + *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true + fail "typed SGLang engine config mount is missing: $typed_engine_mounts" ;; +esac +log "typed SGLang PodLocal Pod persisted with the common MP native sidecar and complete engine wire" + kubectl delete namespace "$CANONICAL_SMOKE_NS" \ --wait=false --ignore-not-found=true >/dev/null 2>&1 || true diff --git a/internal/adapters/builtin/runtime/sglang_lmcache.go b/internal/adapters/builtin/runtime/sglang_lmcache.go index 414c47b9..cb2a307a 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache.go @@ -6,6 +6,7 @@ package runtime import ( "fmt" + "strconv" corev1 "k8s.io/api/core/v1" @@ -144,7 +145,14 @@ func (sglangLMCacheAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a if cache.Spec.LMCache.PodLocal == nil || cache.Spec.LMCache.PodLocal.Server == nil { return fmt.Errorf("SGLang LMCache PodLocal server configuration is missing") } - if _, err := EngineContainerIndexNamed(&pod.Spec, SGLangEngineContainerName); err != nil { + engineIndex, err := EngineContainerIndexNamed(&pod.Spec, SGLangEngineContainerName) + if err != nil { + return err + } + if err := validateSGLangMPPageSize( + pod.Spec.Containers[engineIndex].Args, + effectiveLMCacheChunkSize(cache.Spec.LMCache), + ); err != nil { return err } if findContainerByName(pod.Spec.InitContainers, sglangMPWorkerContainerName) != nil { @@ -153,6 +161,42 @@ func (sglangLMCacheAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a return nil } +// validateSGLangMPPageSize catches a launch-time incompatibility before the +// webhook renders the MP wire. LMCache 0.5.3 also checks the effective page +// size after SGLang has resolved model/backend-specific defaults; that runtime +// check remains authoritative when SGLang rewrites an explicitly declared +// value. Requiring the Pod template to declare --page-size makes the admission +// preflight deterministic instead of guessing from an image tag or a moving +// SGLang default. +func validateSGLangMPPageSize(args []string, chunkSize int32) error { + const pageSizeFlag = "--page-size" + values, malformed := argValues(args, pageSizeFlag) + if malformed { + return fmt.Errorf("SGLang LMCache MP %s is malformed; declare one positive integer value", pageSizeFlag) + } + if len(values) == 0 { + return fmt.Errorf("SGLang LMCache MP engine must explicitly declare %s so chunk-size compatibility can be verified", pageSizeFlag) + } + if len(values) > 1 { + return fmt.Errorf("SGLang LMCache MP %s is duplicated", pageSizeFlag) + } + pageSize, err := strconv.ParseInt(values[0], 10, 32) + if err != nil || pageSize < 1 { + return fmt.Errorf("SGLang LMCache MP %s=%q must be a positive integer", pageSizeFlag, values[0]) + } + if int64(chunkSize)%pageSize != 0 { + return fmt.Errorf("LMCache chunk size %d must be a multiple of SGLang page size %d", chunkSize, pageSize) + } + return nil +} + +func effectiveLMCacheChunkSize(spec *cachev1alpha1.LMCacheEngineSpec) int32 { + if spec != nil && spec.ChunkSizeTokens != nil { + return *spec.ChunkSizeTokens + } + return 256 +} + func injectSGLangLMCachePodLocal(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { return err @@ -162,10 +206,7 @@ func injectSGLangLMCachePodLocal(pod *corev1.PodSpec, binding *backendadapter.Bi return fmt.Errorf("inject SGLang LMCache MP: typed PodLocal server configuration is required") } server := lm.PodLocal.Server - chunkSize := int32(256) - if lm.ChunkSizeTokens != nil { - chunkSize = *lm.ChunkSizeTokens - } + chunkSize := effectiveLMCacheChunkSize(lm) // Compose the common server and SGLang launch surface on one copy. Although // the post-render SGLang upserts cannot fail, keeping one commit point makes diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index fe3be0fe..2b12520e 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -285,7 +285,10 @@ func TestSGLangValidateTypedMPEnginePod(t *testing.T) { runtimeadapter.AnnotationLMCacheConnectorProfile: sglangLMCacheMPConnectorProfile, runtimeadapter.AnnotationLMCacheClientVersion: sglangLMCacheMPClientVersion, }}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName}}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, + Args: []string{"--page-size=1"}, + }}}, } if err := runtimeadapter.ValidateConnectorDeclaration(pod, adapter.ConnectorRequirement(cache)); err != nil { t.Fatalf("ValidateConnectorDeclaration: %v", err) @@ -300,6 +303,42 @@ func TestSGLangValidateTypedMPEnginePod(t *testing.T) { } } +func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + tests := []struct { + name string + args []string + wantErr string + }{ + {name: "pair form", args: []string{"--page-size", "64"}}, + {name: "equals form", args: []string{"--page-size=128"}}, + {name: "missing", wantErr: "explicitly declare --page-size"}, + {name: "not a divisor", args: []string{"--page-size=96"}, wantErr: "chunk size 256 must be a multiple"}, + {name: "zero", args: []string{"--page-size=0"}, wantErr: "positive integer"}, + {name: "not an integer", args: []string{"--page-size=large"}, wantErr: "positive integer"}, + {name: "missing pair value", args: []string{"--page-size", "--model-path", "model"}, wantErr: "malformed"}, + {name: "duplicate", args: []string{"--page-size=1", "--page-size", "64"}, wantErr: "duplicated"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, + Args: tc.args, + }}}} + err := adapter.ValidateMPEnginePod(pod, newTypedSGLangMPBackend()) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("ValidateMPEnginePod: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateMPEnginePod error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + func TestSGLangInjectEngineConfig(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{}) cb := newSGLangBackend(nil) diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index 0cc95860..41778bce 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -326,7 +326,7 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { }, }, Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: "sglang", Image: "sglang:connector-ready", + Name: "sglang", Image: "sglang:connector-ready", Args: []string{"--page-size=1"}, }}}, } if err := mgr.GetClient().Create(ctx, typedPod); err != nil { diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index b2b14332..2c5d998a 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -447,6 +447,7 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { } pod.Annotations[adapterruntime.AnnotationLMCacheConnectorProfile] = "sglang-lmcache-mp-v1" pod.Annotations[adapterruntime.AnnotationLMCacheClientVersion] = "0.5.3" + pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--page-size=1") req := newRequest(t, pod, ns) resp := h.Handle(context.Background(), req) @@ -476,6 +477,26 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-worker") != nil { t.Fatalf("typed wire fell through to legacy worker: %+v", mutated.Spec.InitContainers) } + + // The compatibility guard must fail open atomically: an incompatible page + // size admits the inference Pod but renders neither the server nor half of + // the engine wire. The response message is the actionable admission trace; + // controller status subsequently counts the Pod as uncovered. + incompatible := sglangEnginePod("sg-engine-incompatible", map[string]string{"app": "sglang"}) + incompatible.Annotations = map[string]string{ + adapterruntime.AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", + adapterruntime.AnnotationLMCacheClientVersion: "0.5.3", + } + incompatible.Spec.Containers[0].Args = append(incompatible.Spec.Containers[0].Args, "--page-size=96") + incompatibleReq := newRequest(t, incompatible, ns) + incompatibleResp := h.Handle(context.Background(), incompatibleReq) + if !incompatibleResp.Allowed || len(incompatibleResp.Patches) != 0 { + t.Fatalf("incompatible page size must admit unchanged: Allowed=%v patches=%d result=%+v", + incompatibleResp.Allowed, len(incompatibleResp.Patches), incompatibleResp.Result) + } + if incompatibleResp.Result == nil || !strings.Contains(incompatibleResp.Result.Message, "chunk size 256 must be a multiple") { + t.Fatalf("incompatible page-size diagnostic = %+v", incompatibleResp.Result) + } } func TestHandle_MatchAndInject_SGLangHiCacheWithoutEndpoint(t *testing.T) { diff --git a/test/fixtures/sglang-lmcache/Dockerfile b/test/fixtures/sglang-lmcache/Dockerfile new file mode 100644 index 00000000..ad546875 --- /dev/null +++ b/test/fixtures/sglang-lmcache/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Phase-3 connector-ready SGLang validation fixture. This image belongs to the +# inference owner; CacheBackend never injects or replaces it. Build for +# linux/amd64, push to the GPU test cluster's registry, and record the resulting +# derived digest in the migration roadmap before running the GPU matrix. +FROM lmsysorg/sglang@sha256:1c64fde976bdf0d56474a30bccbcfc19667e5b3ab34c826a534c9d6aaca41212 + +RUN python3 -m pip install --no-cache-dir "lmcache==0.5.3" \ + && python3 -c "import importlib.metadata as m; from lmcache.integration.sglang.multi_process_adapter import LMCacheMPConnector; assert m.version('lmcache') == '0.5.3'" + +LABEL org.opencontainers.image.title="inference-cache SGLang LMCache connector fixture" \ + org.opencontainers.image.description="SGLang v0.5.13.post1-cu130 with LMCache 0.5.3 for PodLocal MP validation" \ + io.inferencecache.lmcache-connector-profile="sglang-lmcache-mp-v1" \ + io.inferencecache.lmcache-client-version="0.5.3" From bd435ede7c408ea173d66e3345ea2378658d19f0 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Mon, 10 Aug 2026 00:05:51 -0700 Subject: [PATCH 04/13] Start vLLM PodLocal MP production baseline Signed-off-by: Yue Sun --- config/samples/README.md | 6 +- ...ebackend-vllm-podlocal-external-redis.yaml | 48 +++ .../cachebackend-vllm-podlocal-host-only.yaml | 38 +++ ...hebackend-vllm-podlocal-managed-redis.yaml | 48 +++ docs/design/cachebackend-api.md | 55 +++- .../lmcache-multiprocess-migration-roadmap.md | 94 +++++- .../scripts/default_install_smoke.sh | 95 ++++++ internal/adapters/builtin/registry.go | 1 + internal/adapters/builtin/registry_test.go | 30 ++ .../builtin/runtime/vllm_lmcache_mp.go | 288 +++++++++++++++++ .../builtin/runtime/vllm_lmcache_mp_test.go | 296 ++++++++++++++++++ .../webhook/pod/envtest_integration_test.go | 95 ++++++ internal/webhook/pod/podinjector_test.go | 101 +++++- .../cachebackend_lmcache_mp_validation.go | 18 +- ...cachebackend_lmcache_mp_validation_test.go | 30 +- .../cachebackend_override_validation_test.go | 14 + .../v1alpha1/cachebackend_validator_test.go | 1 + 17 files changed, 1218 insertions(+), 40 deletions(-) create mode 100644 config/samples/cachebackend-vllm-podlocal-external-redis.yaml create mode 100644 config/samples/cachebackend-vllm-podlocal-host-only.yaml create mode 100644 config/samples/cachebackend-vllm-podlocal-managed-redis.yaml create mode 100644 internal/adapters/builtin/runtime/vllm_lmcache_mp.go create mode 100644 internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go diff --git a/config/samples/README.md b/config/samples/README.md index eb45b51b..1372a572 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -18,7 +18,11 @@ multi-tenant, Namespaces): engine-local example and the typed SGLang PodLocal LMCache examples for [host-only](cachebackend-sglang-podlocal-host-only.yaml), [managed Redis](cachebackend-sglang-podlocal-managed-redis.yaml), and - [external Redis](cachebackend-sglang-podlocal-external-redis.yaml). The + [external Redis](cachebackend-sglang-podlocal-external-redis.yaml), plus the + equivalent typed vLLM PodLocal profiles for + [host-only](cachebackend-vllm-podlocal-host-only.yaml), + [managed Redis](cachebackend-vllm-podlocal-managed-redis.yaml), and + [external Redis](cachebackend-vllm-podlocal-external-redis.yaml). The `recipe-*.yaml` catalog remains the maintained entry point for legacy in-process LMCache scenarios until the repository-wide Phase-5 migration. diff --git a/config/samples/cachebackend-vllm-podlocal-external-redis.yaml b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml new file mode 100644 index 00000000..a84bc17a --- /dev/null +++ b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed vLLM PodLocal LMCache MP with externally managed Redis. Create +# Secret/redis-auth with key password in this namespace before starting +# matching engines. LMCache 0.5.3 RESP supports authentication but not TLS, so +# keep the endpoint on a trusted private network; admission rejects an inert TLS +# block rather than silently sending plaintext. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-podlocal-external-redis +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: vllm-mp-external-redis + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + remoteStorage: + provider: Redis + ownership: External + endpoint: redis.example.internal:6379 + redis: + authentication: + password: + name: redis-auth + key: password + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-vllm-podlocal-host-only.yaml b/config/samples/cachebackend-vllm-podlocal-host-only.yaml new file mode 100644 index 00000000..38f98b8d --- /dev/null +++ b/config/samples/cachebackend-vllm-podlocal-host-only.yaml @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed vLLM PodLocal LMCache MP with no remote tier. Matching engine Pods must +# declare connector profile vllm-lmcache-mp-v1 and LMCache client version 0.5.3. +# CacheBackend injects the MP server sidecar and connector JSON but never +# changes the engine image. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-podlocal-host-only +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: vllm-mp + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml new file mode 100644 index 00000000..89041e13 --- /dev/null +++ b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed vLLM PodLocal LMCache MP with a controller-managed development Redis +# remote tier. Redis is ephemeral soft state and intentionally has one replica. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-podlocal-managed-redis +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/runtime: vllm-mp-managed-redis + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + remoteStorage: + provider: Redis + ownership: Managed + redis: + image: redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 + resources: + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: "2" + memory: 8Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 638da88d..7b0d7b1d 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -188,9 +188,48 @@ Limits-only shapes admit unchanged for any resource — K8s auto-populates `requ **Resource names must match K8s container-resource rules.** `ResourceList` keys are opaque map keys at the CRD-schema layer; an invalid name like `"foo"` or `""` persists in etcd and only fails when the apiserver later rejects the child pod. The validating webhook (`rejectInvalidResourceNames`) applies the same rules the apiserver applies to a `Container.Resources` map: standard names (`cpu`, `memory`, `ephemeral-storage`) admit unconditionally; a `hugepages-` name admits only when the size suffix parses as a strictly-positive `resource.Quantity` (e.g. `"hugepages-2Mi"`, `"hugepages-1Gi"` — a bare `"hugepages-"` or non-numeric `"hugepages-nope"` is rejected because the apiserver requires the size token); any other name must be **third-party vendor-prefixed** (e.g. `"nvidia.com/gpu"`) and pass `IsQualifiedName`. A bare unqualified `"foo"` is rejected even though `IsQualifiedName` alone admits it, because the apiserver's container-resource layer requires extended resources to carry a vendor identity. Names under the **K8s-reserved prefixes `kubernetes.io/` and `requests.kubernetes.io/`** are also rejected — those prefixes are reserved for native resources, so extended resources may not use them. The rejection names the offending key so multi-key errors surface together. **Inert without a controller-managed workload.** Host-only, externally owned, -and `SGLangHiCache` configurations provision no cache-server workload of their -own. HiCache host memory belongs to the user-owned engine container and must be -sized on that workload instead. +and `SGLangHiCache` configurations provision no provider Deployment or Service. +Typed PodLocal LMCache still injects its server into each matching engine Pod as +a native sidecar. HiCache host memory belongs to the user-owned engine container +and must be sized on that workload instead. + +### vLLM typed PodLocal LMCache MP support + +The typed shape `spec.runtime: VLLM`, `spec.type: LMCache`, and +`spec.lmCache.topology: PodLocal` selects a dedicated MP adapter; it does not +reuse the legacy `LMCacheConnectorV1` / `lm://` path. The engine image remains +owned by the inference runtime. To make that image's capability explicit, each +matching Pod must declare: + +```yaml +metadata: + annotations: + inferencecache.io/lmcache-connector-profile: vllm-lmcache-mp-v1 + inferencecache.io/lmcache-client-version: "0.5.3" +``` + +The webhook injects a digest-pinned `lmcache-mp-server` native sidecar and adds +the following vLLM launch contract: + +- `--kv-transfer-config` selects `LMCacheMPConnector` through + `lmcache.integration.vllm.lmcache_mp_connector`, points it at + `tcp://127.0.0.1:`, and maps the integration role to + `kv_consumer`, `kv_producer`, or `kv_both`; +- `--disable-hybrid-kv-cache-manager` is required by the initial pinned + profile; +- `PYTHONHASHSEED=0` stabilizes vLLM's cross-process hash chain; +- `INFERENCECACHE_FAIL_OPEN` mirrors the API setting, although runtime-native + failure behavior still requires GPU validation. + +The typed adapter accepts host-only or RESP bindings. Redis credentials are +mounted into the MP server from `SecretKeyRef`; they are not copied into the +vLLM container. LMCache 0.5.3 TLS and logical-database selection remain rejected +because that RESP adapter cannot consume them. The initial adapter admits TP +but rejects PP/DP greater than one and external multi-process DP flags. These +checks and persisted webhook injection are covered without GPU; the pinned +vLLM image/version, KV reuse, TP determinism, and failure recovery remain Phase +4 runtime gates. Canonical examples are the three +`config/samples/cachebackend-vllm-podlocal-*.yaml` files. ### SGLang engine support @@ -213,7 +252,7 @@ adapter configures the node-local MP worker and accepts either no binding Redis workload only when `spec.remoteStorage` explicitly selects `provider: Redis`, `ownership: Managed`. -> **Cluster prerequisite — Kubernetes ≥ 1.29 (REQUIRED for the SGLang MP wire).** The MP worker is injected as a **native sidecar** — an `initContainers` entry with `restartPolicy: Always`, which K8s only understands from 1.29 (beta, on by default; stable 1.33). On an older cluster the apiserver does not recognize that field, so a `(sglang, LMCache)` engine pod **fails admission** (or the worker degrades to a plain init container that exits before the engine starts) rather than failing open — the one place this pair has a hard cluster-version floor. vLLM+LMCache and the routing-only path have no such floor. There is no in-webhook version gate today; operators on the SGLang pair must run 1.29+. +> **Cluster prerequisite — Kubernetes ≥ 1.29 (REQUIRED for typed PodLocal LMCache).** The MP server is injected as a **native sidecar** — an `initContainers` entry with `restartPolicy: Always`, which K8s only understands from 1.29 (beta, on by default; stable 1.33). On an older cluster the apiserver does not recognize that field, so a typed SGLang or vLLM PodLocal engine pod fails admission (or the server degrades to a plain init container that exits before the engine starts). There is no in-webhook version gate today; operators using typed PodLocal LMCache must run 1.29+. > **Two more caveats on the SGLang support surface** (details below): (1) server-derived `LookupRoute` with raw `token_ids`/`prompt_text` only hits when the server's single global `--engine-block-size` matches SGLang's page size (see the "Block-size alignment" note later in this section); gateways that send pre-computed `prefix_hash`/`block_hashes` are unaffected. (2) The `lmcache-kernel-check` init container is vLLM-only today (the SGLang adapter does not implement `InitContainerProvider`), so `EngineKernelsHealthy` is not published for SGLang pods. @@ -730,7 +769,7 @@ flag/env and the adapter. Warning-only would let a user silently un-wire the integration and discover it via a crashed engine; the hard-reject keeps the breadcrumb at admission time. -The vLLM+LMCache adapter (`internal/adapters/builtin/runtime/vllm_lmcache.go`) reserves the args/env the integration cannot function without: +The legacy vLLM+LMCache adapter (`internal/adapters/builtin/runtime/vllm_lmcache.go`) reserves the args/env the integration cannot function without: - `ReservedArgs()`: `--kv-transfer-config` (the LMCache connector wiring). - `ReservedEnv()`: `VLLM_USE_V1` (selects the engine codepath the connector targets), `LMCACHE_REMOTE_URL` (the resolved cache endpoint), `INFERENCECACHE_FAIL_OPEN` (mirror of `spec.integration.failOpen` — overriding it would silently desync the pod from the CR contract), `PYTHONHASHSEED` (pins the deterministic `NONE_HASH` so LMCache reload matches under TP>1 — overriding or suppressing it silently 0-hits reload). @@ -743,6 +782,12 @@ an override that would remove connector wiring regardless of provider ownership. See [Mooncake provider configuration](#mooncake-provider-configuration). +The typed PodLocal vLLM MP adapter reserves a narrower and different set: +`ReservedArgs()` = `--kv-transfer-config`, +`--disable-hybrid-kv-cache-manager`; `ReservedEnv()` = `PYTHONHASHSEED`, +`INFERENCECACHE_FAIL_OPEN`. It does not inject or reserve +`LMCACHE_REMOTE_URL`, `VLLM_USE_V1`, or the legacy serde/local-CPU variables. + The SGLang+LMCache adapter (`internal/adapters/builtin/runtime`) reserves a **different** set, because SGLang's engine-side wire is the LMCache MP wire, not the `lm://` one (see [SGLang engine support](#sglang-engine-support)): `ReservedArgs()` = `--enable-lmcache`, `--lmcache-config-file`; `ReservedEnv()` = `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. Suppressing `--lmcache-config-file` un-wires MP mode (the engine aborts at startup without it), hence its reservation. In MP mode the lm:// `LMCACHE_REMOTE_URL` is neither injected nor reserved, and `VLLM_USE_V1` / `PYTHONHASHSEED` are never injected for SGLang. Reservation is per-adapter precisely so each engine guards only the flags/env its own integration cannot function without. `LMCACHE_CHUNK_SIZE`, `LMCACHE_REMOTE_SERDE`, `LMCACHE_LOCAL_CPU`, `LMCACHE_MAX_LOCAL_CPU_SIZE` are deliberately NOT reserved — they are perf/mode tunables the operator may legitimately want to change. Canonical chunk size, serializer, and host-memory capacity use `spec.lmCache`; `engineOverrides.env` remains the engine-agnostic seam for explicit environment-level tuning. diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 518e4051..53818263 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -357,7 +357,7 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 1 | MP-only API and admission/status contracts | Phase 0 | complete | | 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | complete | | 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | in progress — non-GPU baseline complete; GPU matrix pending | -| 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | not started | +| 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | in progress — non-GPU control-plane baseline complete; engine/GPU matrix pending | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | | 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 finding | | 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | not started | @@ -763,32 +763,94 @@ is complete. No Phase 3 GPU data-path or runtime-failure result is claimed yet. Provide the complete replacement for the current vLLM IP path before any IP consumer is forced to migrate. +Current state: the dedicated adapter and non-GPU control-plane baseline are +complete. No vLLM process has loaded the connector and no GPU KV operation or +runtime failure/recovery result is claimed yet. Phase 3's remaining GPU work +does not block this independent adapter work, but both phases retain their GPU +exit gates. + +### Completed non-GPU validation + +- [x] Inspect the exact LMCache 0.5.3 source and its operator golden config. + The required vLLM wire is the external `LMCacheMPConnector` module + `lmcache.integration.vllm.lmcache_mp_connector`, with + `lmcache.mp.host` and `lmcache.mp.port` in + `kv_connector_extra_config`. This is source-level contract evidence, not + proof that the pinned vLLM reference image contains a compatible vLLM. +- [x] Add a dedicated typed vLLM MP adapter and register it before the legacy + vLLM adapter. Registry tests prove typed PodLocal objects select MP while + topology-less objects retain the legacy adapter during the compatibility + window. +- [x] Require the engine owner to declare connector profile + `vllm-lmcache-mp-v1` and LMCache client version `0.5.3`; a missing or + mismatched declaration admits the Pod unchanged with a fail-open + diagnostic rather than guessing from its image name. +- [x] Render the exact connector module, loopback address, configured port, and + role mapping (`ReadOnly`/`WriteOnly`/`ReadWrite` to + `kv_consumer`/`kv_producer`/`kv_both`). +- [x] Reuse the engine-neutral PodLocal renderer for the digest-pinned native + sidecar, bounded `/dev/shm`, probes, resources, and optional RESP L2. + Redis username/password remain `SecretKeyRef` values on the MP server and + are never copied into engine arguments or environment variables. +- [x] Remove legacy `LMCACHE_REMOTE_URL`, serde, chunk, and local-CPU env from + the typed wire; vLLM consumes connector JSON instead of SGLang's client + YAML volume. +- [x] Inject `PYTHONHASHSEED=0`, reserve the connector/hybrid arguments and + correctness-critical env, and preserve atomic/idempotent mutation. +- [x] Accept positive TP declarations, reject PP or DP greater than one, + external/multi-process DP flags, malformed/duplicate parallel flags, and + inject `--disable-hybrid-kv-cache-manager` for the initial profile. +- [x] Add typed host-only, managed-Redis, and external-Redis vLLM samples. All + three pass real envtest API-server admission; the external profile's + Secret-backed authentication is accepted while TLS/database remain + explicitly unsupported by LMCache 0.5.3. +- [x] Persist a typed vLLM Pod through envtest kube-apiserver/etcd and verify + the external connector module, loopback MP address, deterministic hash + seed, and common native-sidecar schema. +- [x] Install the controller and webhooks in a Kubernetes 1.32 kind cluster, + create a connector-declared vLLM Pod through the live mutating webhook, + read the persisted object back from etcd, and verify the exact MP wire. + An impossible node selector kept the Pod unscheduled, so neither the + engine nor MP server ran and no engine image was pulled. +- [x] Pass the non-GPU regression gates: full Go tests, focused envtest, + sample verification (27 pass, 2 intentional skips), default-install + smoke, Go vet, Prometheus rules, naming/internal-reference checks, docs + sync, and REUSE lint. + +The pinned 29 GiB vLLM reference image was deliberately not pulled or built for +this non-GPU baseline. Recording its exact `vllm.__version__`, importing the +connector inside that image, and exercising the engine CLI remain the first +runtime preflight before GPU testing. + ### Engine wire -- [ ] Add a dedicated vLLM MP adapter; do not mutate the legacy adapter in place. -- [ ] Render `LMCacheMPConnector` with: - - `kv_connector_module_path` selecting the pinned implementation required by - D11; - - `kv_role` derived from `integration.role`; - - `lmcache.mp.host=127.0.0.1`; - - the configured MP port; - - validated MQ timeout/heartbeat settings when exposed; - - runtime-native load-failure recompute/fail-open behavior. -- [ ] Preserve `PYTHONHASHSEED=0` across scheduler and worker processes. -- [ ] Reserve only correctness-critical args/env owned by the adapter. -- [ ] Reject hybrid/parallelism combinations not supported by the pinned - vLLM/LMCache tuple. +- [x] Add a dedicated vLLM MP adapter; do not mutate the legacy adapter in place. +- [x] Render the connector module path, role, loopback host, and configured MP + port in `LMCacheMPConnector` JSON. +- [ ] Add MQ timeout/heartbeat fields only when a pinned public configuration + surface exists. LMCache 0.5.3 currently supplies internal defaults, so + the API and renderer do not invent unsupported knobs. +- [ ] Prove runtime-native load-failure recompute/fail-open behavior. +- [x] Inject `PYTHONHASHSEED=0`; cross-process hash determinism still requires + the GPU/runtime test below. +- [x] Reserve only correctness-critical args/env owned by the typed adapter. +- [ ] Complete hybrid/parallelism classification for the pinned tuple. The + deterministic Pod-visible PP/DP restrictions and hybrid-manager guard are + implemented; MLA/model-specific behavior remains runtime validation. ### Functional scope - [ ] vLLM + PodLocal + no remote L3. - [ ] vLLM + PodLocal + managed Redis development profile. - [ ] vLLM + PodLocal + external Redis production profile. -- [ ] ReadOnly, WriteOnly, and ReadWrite roles. +- [x] ReadOnly, WriteOnly, and ReadWrite connector JSON rendering. Runtime KV + behavior remains part of the GPU matrix. - [ ] TP=1 and TP=2; TP=4 before recommending the topology for common multi-GPU production workloads. - [ ] Multi-server, DP + multi-server, and unsupported PP/MLA combinations are - rejected, not silently attempted. + rejected, not silently attempted. PP>1, DP>1, and external DP flags are + already rejected from Pod-visible arguments; multi-server and MLA + classification remain pending. ### Correctness and failure tests diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index e92eb6a4..a3e8ef7b 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -82,6 +82,10 @@ # pinned to an impossible node selector. The smoke reads the persisted Pod # back and asserts the common lmcache-mp-server native sidecar, probes, # resources, shared mounts, engine flags/env, and injection identity. +# 8e. Typed vLLM PodLocal admission: a matching, connector-declared vLLM Pod +# is persisted through the same live webhook and carries the dedicated +# LMCacheMPConnector module path, loopback MP endpoint, deterministic hash +# seed, hybrid-manager guard, and no legacy lm:// environment. # 9. Canonical External ownership end-to-end: applying the committed # config/samples/cachebackend-external.yaml drives the CacheBackend # mutating webhook default (spec.replicas=1), renders NO @@ -376,6 +380,8 @@ CANONICAL_HOST_ONLY_CB="cachebackend-sglang-host-only" CANONICAL_REDIS_CB="cachebackend-sglang" CANONICAL_TYPED_CB="sglang-podlocal-host-only" CANONICAL_TYPED_POD="sglang-podlocal-admission" +CANONICAL_TYPED_VLLM_CB="vllm-podlocal-host-only" +CANONICAL_TYPED_VLLM_POD="vllm-podlocal-admission" # Events-only-backend smoke fixture identifiers. Declared up front so the # diagnostics helper can reference them even if the smoke aborts before the @@ -456,6 +462,8 @@ collect_diagnostics() { >"$LOG_DIR/canonical-provider-workloads.yaml" 2>&1 || true kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml \ >"$LOG_DIR/sglang-podlocal-admission.yaml" 2>&1 || true + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml \ + >"$LOG_DIR/vllm-podlocal-admission.yaml" 2>&1 || true # External-backend smoke artefacts. Best-effort — the CR/pod may not # exist if the smoke aborted before that section. kubectl get cb -A -o wide \ @@ -2353,6 +2361,93 @@ case " $typed_engine_mounts " in esac log "typed SGLang PodLocal Pod persisted with the common MP native sidecar and complete engine wire" +kubectl -n "$CANONICAL_SMOKE_NS" apply \ + -f config/samples/cachebackend-vllm-podlocal-host-only.yaml >/dev/null \ + || fail "typed vLLM PodLocal CacheBackend sample failed to apply" + +typed_vllm_pod="$(mktemp "$tmpdir/vllm-podlocal-admission.XXXXXX.yaml")" +cat >"$typed_vllm_pod" <<'EOF' +apiVersion: v1 +kind: Pod +metadata: + name: vllm-podlocal-admission + labels: + inferencecache.io/runtime: vllm-mp + annotations: + inferencecache.io/lmcache-connector-profile: vllm-lmcache-mp-v1 + inferencecache.io/lmcache-client-version: "0.5.3" +spec: + nodeSelector: + inferencecache.io/install-smoke-never-schedule: "true" + containers: + - name: vllm + image: example.invalid/vllm-lmcache@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + command: ["python3", "-m", "vllm.entrypoints.openai.api_server"] + args: + - --model=meta-llama/Meta-Llama-3-8B-Instruct + - --tensor-parallel-size=2 +EOF +kubectl -n "$CANONICAL_SMOKE_NS" create -f "$typed_vllm_pod" >/dev/null \ + || fail "matching typed vLLM Pod did not pass the live mutating webhook" + +typed_vllm_injected_by="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true)" +typed_vllm_metrics_label="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.metadata.labels.inferencecache\.io/lmcache-mp-metrics}' 2>/dev/null || true)" +typed_vllm_server_image="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].image}' 2>/dev/null || true)" +typed_vllm_server_restart="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].restartPolicy}' 2>/dev/null || true)" +typed_vllm_engine_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="vllm")].args}' 2>/dev/null || true)" +typed_vllm_config="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o json \ + | jq -r '.spec.containers[] | select(.name == "vllm") | .args as $args | ($args | index("--kv-transfer-config")) as $index | if $index == null then "" else $args[$index + 1] end')" +typed_vllm_hash_seed="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="PYTHONHASHSEED")].value}' 2>/dev/null || true)" +typed_vllm_legacy_url="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="LMCACHE_REMOTE_URL")].value}' 2>/dev/null || true)" +typed_vllm_engine_mounts="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ + -o jsonpath='{.spec.containers[?(@.name=="vllm")].volumeMounts[*].mountPath}' 2>/dev/null || true)" + +if [ "$typed_vllm_injected_by" != "$CANONICAL_SMOKE_NS/$CANONICAL_TYPED_VLLM_CB" ] || \ + [ "$typed_vllm_metrics_label" != "true" ] || \ + [ "$typed_vllm_server_image" != "$expected_mp_image" ] || \ + [ "$typed_vllm_server_restart" != "Always" ] || \ + [ "$typed_vllm_hash_seed" != "0" ] || \ + [ -n "$typed_vllm_legacy_url" ]; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true + fail "typed vLLM persisted wire metadata is wrong: injectedBy=$typed_vllm_injected_by metrics=$typed_vllm_metrics_label image=$typed_vllm_server_image restart=$typed_vllm_server_restart hashSeed=$typed_vllm_hash_seed legacyURL=$typed_vllm_legacy_url" +fi +if ! jq -e ' + .kv_connector == "LMCacheMPConnector" and + .kv_connector_module_path == "lmcache.integration.vllm.lmcache_mp_connector" and + .kv_role == "kv_both" and + .kv_connector_extra_config["lmcache.mp.host"] == "tcp://127.0.0.1" and + .kv_connector_extra_config["lmcache.mp.port"] == "5555" + ' >/dev/null <<<"$typed_vllm_config"; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true + fail "typed vLLM connector JSON is wrong: $typed_vllm_config" +fi +if ! jq -e ' + index("--tensor-parallel-size=2") != null and + index("--disable-hybrid-kv-cache-manager") != null and + index("--kv-transfer-config") != null + ' >/dev/null <<<"$typed_vllm_engine_args"; then + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true + fail "typed vLLM engine args are incomplete: $typed_vllm_engine_args" +fi +case " $typed_vllm_engine_mounts " in + *" /dev/shm "*) : ;; + *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true + fail "typed vLLM engine /dev/shm mount is missing: $typed_vllm_engine_mounts" ;; +esac +case " $typed_vllm_engine_mounts " in + *" /var/run/inference-cache/lmcache "*) + kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true + fail "typed vLLM unexpectedly received the SGLang YAML config mount: $typed_vllm_engine_mounts" ;; +esac +log "typed vLLM PodLocal Pod persisted with the dedicated external MP connector wire" + kubectl delete namespace "$CANONICAL_SMOKE_NS" \ --wait=false --ignore-not-found=true >/dev/null 2>&1 || true diff --git a/internal/adapters/builtin/registry.go b/internal/adapters/builtin/registry.go index 8068c79a..db77b842 100644 --- a/internal/adapters/builtin/registry.go +++ b/internal/adapters/builtin/registry.go @@ -34,6 +34,7 @@ func New(opts Options) Registries { PolicyServerGRPCAddress: opts.PolicyServerGRPCAddress, } runtimeRegistry := adapterruntime.NewRegistry() + runtimeRegistry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(subscriber)) runtimeRegistry.Register(builtinruntime.NewVLLMLMCacheAdapter(subscriber)) runtimeRegistry.Register(builtinruntime.NewSGLangLMCacheAdapter(subscriber)) runtimeRegistry.Register(builtinruntime.NewSGLangHiCacheAdapter(subscriber)) diff --git a/internal/adapters/builtin/registry_test.go b/internal/adapters/builtin/registry_test.go index aabc0c69..15073274 100644 --- a/internal/adapters/builtin/registry_test.go +++ b/internal/adapters/builtin/registry_test.go @@ -37,6 +37,36 @@ func TestNewIncludesEveryShippingRuntimeAdapter(t *testing.T) { } } +func TestNewSelectsTypedVLLMMPBeforeLegacyAdapter(t *testing.T) { + t.Parallel() + + registry := New(Options{}).Runtime + typed := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{Topology: cachev1alpha1.LMCacheTopologyPodLocal}, + }} + adapter, err := registry.Select(adapterruntime.RuntimeVLLM, typed) + if err != nil { + t.Fatalf("Select typed vLLM adapter: %v", err) + } + if _, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter); !ok { + t.Fatalf("typed vLLM adapter = %T, want LMCacheMPRuntimeAdapter", adapter) + } + + legacy := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + }} + adapter, err = registry.Select(adapterruntime.RuntimeVLLM, legacy) + if err != nil { + t.Fatalf("Select legacy vLLM adapter: %v", err) + } + if _, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter); ok { + t.Fatalf("legacy vLLM adapter = %T, unexpectedly implements LMCacheMPRuntimeAdapter", adapter) + } +} + func TestNewIncludesShippingStorageProviders(t *testing.T) { t.Parallel() diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go new file mode 100644 index 00000000..243c8af6 --- /dev/null +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go @@ -0,0 +1,288 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" + runtimeadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" +) + +const ( + vllmLMCacheMPConnectorProfile = "vllm-lmcache-mp-v1" + vllmLMCacheMPClientVersion = "0.5.3" + + vllmLMCacheMPConnectorName = "LMCacheMPConnector" + vllmLMCacheMPConnectorModulePath = "lmcache.integration.vllm.lmcache_mp_connector" + vllmDisableHybridKVCacheArg = "--disable-hybrid-kv-cache-manager" +) + +// vllmLMCacheMPAdapter is the typed PodLocal vLLM adapter. It embeds the +// legacy adapter only to reuse engine-neutral observation and kernel-check +// providers; selection and engine injection are implemented independently so +// the legacy LMCacheConnectorV1/IP wire cannot leak into the MP path. +type vllmLMCacheMPAdapter struct { + vllmLMCacheAdapter +} + +// NewVLLMLMCacheMPAdapter returns the explicit typed PodLocal adapter. Register +// it before NewVLLMLMCacheAdapter because both advertise the canonical +// vllm/LMCache pair and the registry selects the first matching adapter. +func NewVLLMLMCacheMPAdapter(subscriber SubscriberConfig) runtimeadapter.KVCacheRuntimeAdapter { + return vllmLMCacheMPAdapter{vllmLMCacheAdapter: vllmLMCacheAdapter{subscriber: subscriber}} +} + +func (vllmLMCacheMPAdapter) Supports(runtime runtimeadapter.RuntimeID, cache *cachev1alpha1.CacheBackend) bool { + return cache != nil && + runtime == runtimeadapter.RuntimeVLLM && + cache.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && + cache.Spec.LMCache != nil && + cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal +} + +func (vllmLMCacheMPAdapter) SupportsBinding(binding *backendadapter.Binding) bool { + return binding == nil || binding.Protocol == backendadapter.ProtocolRESP +} + +func (vllmLMCacheMPAdapter) ConnectorRequirement(*cachev1alpha1.CacheBackend) runtimeadapter.LMCacheConnectorRequirement { + return runtimeadapter.LMCacheConnectorRequirement{ + Profile: vllmLMCacheMPConnectorProfile, + ClientVersion: vllmLMCacheMPClientVersion, + } +} + +// ValidateMPEnginePod rejects only constraints that can be classified from the +// concrete Pod. The pinned LMCache connector fixes one MP server per vLLM +// instance, and the initial production profile does not claim pipeline or +// multi-process data parallelism. Tensor parallelism remains valid and is GPU +// exercised at TP=1/2 before this phase exits. +func (vllmLMCacheMPAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1alpha1.CacheBackend) error { + if pod == nil { + return fmt.Errorf("vLLM LMCache MP engine pod is nil") + } + if cache == nil || cache.Spec.LMCache == nil { + return fmt.Errorf("vLLM LMCache MP CacheBackend configuration is missing") + } + lm := cache.Spec.LMCache + if lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal { + return fmt.Errorf("vLLM LMCache MP topology %q is not implemented; want %q", + lm.Topology, cachev1alpha1.LMCacheTopologyPodLocal) + } + if lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("vLLM LMCache PodLocal server configuration is missing") + } + engineIndex, err := EngineContainerIndexNamed(&pod.Spec, EngineContainerName) + if err != nil { + return err + } + args := pod.Spec.Containers[engineIndex].Args + if _, err := vllmPositiveParallelSize(args, []string{"--tensor-parallel-size", "-tp"}, 1); err != nil { + return fmt.Errorf("vLLM LMCache MP tensor parallelism: %w", err) + } + pp, err := vllmPositiveParallelSize(args, []string{"--pipeline-parallel-size", "-pp"}, 1) + if err != nil { + return fmt.Errorf("vLLM LMCache MP pipeline parallelism: %w", err) + } + if pp != 1 { + return fmt.Errorf("vLLM LMCache MP pipeline parallel size %d is not supported by the initial PodLocal profile; use 1", pp) + } + dp, err := vllmPositiveParallelSize(args, []string{"--data-parallel-size", "-dp"}, 1) + if err != nil { + return fmt.Errorf("vLLM LMCache MP data parallelism: %w", err) + } + if dp != 1 { + return fmt.Errorf("vLLM LMCache MP data parallel size %d is not supported by the initial PodLocal profile; use 1", dp) + } + for _, flag := range []string{ + "--data-parallel-rank", + "--data-parallel-start-rank", + "--data-parallel-size-local", + "--data-parallel-address", + "--data-parallel-rpc-port", + } { + if hasArg(args, flag) { + return fmt.Errorf("vLLM LMCache MP multi-process data parallel flag %s is not supported by the initial PodLocal profile", flag) + } + } + if err := validateVLLMBooleanArg(args, vllmDisableHybridKVCacheArg); err != nil { + return err + } + values, malformed := argValues(args, defaultEngineKVTransferConfigArg) + if malformed || len(values) > 1 { + return fmt.Errorf("vLLM LMCache MP %s must appear at most once with one JSON value", defaultEngineKVTransferConfigArg) + } + return nil +} + +func vllmPositiveParallelSize(args, flags []string, fallback int64) (int64, error) { + var values []string + var seenFlags []string + for index := 0; index < len(args); index++ { + arg := args[index] + for _, flag := range flags { + switch { + case arg == flag: + if index+1 >= len(args) || strings.HasPrefix(args[index+1], "-") { + return 0, fmt.Errorf("%s is malformed; declare one positive integer value", flag) + } + values = append(values, args[index+1]) + seenFlags = append(seenFlags, flag) + index++ + case strings.HasPrefix(arg, flag+"="): + value := strings.TrimPrefix(arg, flag+"=") + if value == "" { + return 0, fmt.Errorf("%s is malformed; declare one positive integer value", flag) + } + values = append(values, value) + seenFlags = append(seenFlags, flag) + } + } + } + if len(values) == 0 { + return fallback, nil + } + if len(values) > 1 { + return 0, fmt.Errorf("parallel-size aliases are duplicated: %s", strings.Join(seenFlags, ", ")) + } + value, err := strconv.ParseInt(values[0], 10, 32) + if err != nil || value < 1 { + return 0, fmt.Errorf("%s=%q must be a positive integer", seenFlags[0], values[0]) + } + return value, nil +} + +func validateVLLMBooleanArg(args []string, flag string) error { + count := 0 + for index, arg := range args { + switch { + case arg == flag: + count++ + if index+1 < len(args) { + next := strings.ToLower(args[index+1]) + if next == "true" || next == "false" || next == "0" || next == "1" { + return fmt.Errorf("vLLM LMCache MP %s is a boolean flag and must not carry value %q", flag, args[index+1]) + } + } + case strings.HasPrefix(arg, flag+"="): + return fmt.Errorf("vLLM LMCache MP %s is a boolean flag and must not carry a value", flag) + } + } + if count > 1 { + return fmt.Errorf("vLLM LMCache MP %s is duplicated", flag) + } + return nil +} + +type vllmMPKVTransferConfig struct { + Connector string `json:"kv_connector"` + ConnectorModule string `json:"kv_connector_module_path"` + Role string `json:"kv_role"` + ExtraConfig vllmMPConnectorExtraConfig `json:"kv_connector_extra_config"` +} + +type vllmMPConnectorExtraConfig struct { + Host string `json:"lmcache.mp.host"` + Port string `json:"lmcache.mp.port"` +} + +func vllmMPKVTransferConfigJSON(role cachev1alpha1.CacheBackendIntegrationRole, port int32) (string, error) { + kvRole := "" + switch role { + case cachev1alpha1.CacheBackendIntegrationRoleReadOnly: + kvRole = kvRoleConsumer + case cachev1alpha1.CacheBackendIntegrationRoleWriteOnly: + kvRole = kvRoleProducer + case "", cachev1alpha1.CacheBackendIntegrationRoleReadWrite: + kvRole = kvRoleBoth + default: + return "", fmt.Errorf("vLLM LMCache MP integration role %q is unsupported", role) + } + raw, err := json.Marshal(vllmMPKVTransferConfig{ + Connector: vllmLMCacheMPConnectorName, + ConnectorModule: vllmLMCacheMPConnectorModulePath, + Role: kvRole, + ExtraConfig: vllmMPConnectorExtraConfig{ + Host: "tcp://127.0.0.1", + Port: strconv.FormatInt(int64(port), 10), + }, + }) + if err != nil { + return "", fmt.Errorf("marshal vLLM LMCache MP connector config: %w", err) + } + return string(raw), nil +} + +func (vllmLMCacheMPAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { + if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { + return err + } + lm := cache.Spec.LMCache + if lm == nil || lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal || lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("inject vLLM LMCache MP: typed PodLocal server configuration is required") + } + if !(vllmLMCacheMPAdapter{}).SupportsBinding(binding) { + return fmt.Errorf("vLLM LMCache MP adapter does not support remote binding protocol %q", binding.Protocol) + } + server := lm.PodLocal.Server + configJSON, err := vllmMPKVTransferConfigJSON(IntegrationRole(cache), server.Port) + if err != nil { + return err + } + + work := pod.DeepCopy() + if _, err := renderLMCachePodLocalServer(work, EngineContainerName, lmCacheMPServerConfig{ + Image: server.Image, + Port: server.Port, + ChunkSizeTokens: effectiveLMCacheChunkSize(lm), + L1Capacity: server.L1Capacity, + MaxWorkers: server.MaxWorkers, + Resources: server.Resources, + Binding: binding, + WriteClientConfig: false, + }); err != nil { + return err + } + engineIndex, err := EngineContainerIndexNamed(work, EngineContainerName) + if err != nil { + return err + } + engine := &work.Containers[engineIndex] + engine.Args = UpsertArgPair(engine.Args, defaultEngineKVTransferConfigArg, configJSON) + engine.Args = UpsertFlag(engine.Args, vllmDisableHybridKVCacheArg) + for _, name := range []string{ + EnvLMCacheRemoteURL, + EnvLMCacheRemoteSerde, + EnvLMCacheChunkSize, + EnvLMCacheLocalCPU, + EnvLMCacheMaxLocalCPU, + } { + engine.Env = removeEnv(engine.Env, name) + } + engine.Env = removeEnv(engine.Env, EnvPythonHashSeed) + engine.Env = append(engine.Env, corev1.EnvVar{Name: EnvPythonHashSeed, Value: defaultPythonHashSeed}) + engine.Env = removeEnv(engine.Env, EnvInferenceCacheFailOpen) + engine.Env = append(engine.Env, corev1.EnvVar{Name: EnvInferenceCacheFailOpen, Value: FailOpenString(cache)}) + + *pod = *work + return nil +} + +func (vllmLMCacheMPAdapter) ReservedArgs() []string { + return []string{defaultEngineKVTransferConfigArg, vllmDisableHybridKVCacheArg} +} + +func (vllmLMCacheMPAdapter) ReservedEnv() []string { + return []string{EnvPythonHashSeed, EnvInferenceCacheFailOpen} +} + +var _ runtimeadapter.KVCacheRuntimeAdapter = vllmLMCacheMPAdapter{} +var _ runtimeadapter.LMCacheMPRuntimeAdapter = vllmLMCacheMPAdapter{} diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go new file mode 100644 index 00000000..f55cd0ff --- /dev/null +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" + runtimeadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" +) + +func newTypedVLLMMPBackend() *cachev1alpha1.CacheBackend { + chunkSize := int32(256) + return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + ChunkSizeTokens: &chunkSize, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: testLMCacheServerImage, + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + }, + }}, + }, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ + Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, + }, + }, + } +} + +func newVLLMMPEnginePod(args ...string) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: EngineContainerName, + Image: "vllm:connector-ready", + Args: args, + }}}} +} + +func TestVLLMLMCacheMPRegistrySelectionDoesNotChangeLegacy(t *testing.T) { + registry := runtimeadapter.NewRegistry() + registry.Register(NewVLLMLMCacheMPAdapter(SubscriberConfig{})) + registry.Register(NewVLLMLMCacheAdapter(SubscriberConfig{})) + + typed, err := registry.Select(runtimeadapter.RuntimeVLLM, newTypedVLLMMPBackend()) + if err != nil { + t.Fatalf("select typed adapter: %v", err) + } + if _, ok := typed.(vllmLMCacheMPAdapter); !ok { + t.Fatalf("typed adapter = %T, want vllmLMCacheMPAdapter", typed) + } + + legacy, err := registry.Select(runtimeadapter.RuntimeVLLM, newLMCacheBackend(nil)) + if err != nil { + t.Fatalf("select legacy adapter: %v", err) + } + if _, ok := legacy.(vllmLMCacheAdapter); !ok { + t.Fatalf("legacy adapter = %T, want vllmLMCacheAdapter", legacy) + } +} + +func TestVLLMLMCacheMPConnectorRequirement(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + requirement := adapter.ConnectorRequirement(newTypedVLLMMPBackend()) + if requirement.Profile != "vllm-lmcache-mp-v1" || requirement.ClientVersion != "0.5.3" { + t.Fatalf("connector requirement = %+v", requirement) + } +} + +func TestVLLMLMCacheMPReservedSurface(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + if got, want := adapter.ReservedArgs(), []string{defaultEngineKVTransferConfigArg, vllmDisableHybridKVCacheArg}; !reflect.DeepEqual(got, want) { + t.Fatalf("ReservedArgs = %v, want %v", got, want) + } + if got, want := adapter.ReservedEnv(), []string{EnvPythonHashSeed, EnvInferenceCacheFailOpen}; !reflect.DeepEqual(got, want) { + t.Fatalf("ReservedEnv = %v, want %v", got, want) + } +} + +func TestVLLMLMCacheMPValidateEngineParallelism(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + tests := []struct { + name string + args []string + wantErr string + }{ + {name: "defaults"}, + {name: "TP pair", args: []string{"--tensor-parallel-size", "2"}}, + {name: "TP short equals", args: []string{"-tp=2"}}, + {name: "TP duplicate aliases", args: []string{"--tensor-parallel-size=2", "-tp", "2"}, wantErr: "duplicated"}, + {name: "TP malformed", args: []string{"--tensor-parallel-size"}, wantErr: "malformed"}, + {name: "TP zero", args: []string{"--tensor-parallel-size=0"}, wantErr: "positive integer"}, + {name: "PP two", args: []string{"--pipeline-parallel-size=2"}, wantErr: "pipeline parallel size 2"}, + {name: "DP two", args: []string{"-dp", "2"}, wantErr: "data parallel size 2"}, + {name: "external DP rank", args: []string{"--data-parallel-rank=0"}, wantErr: "multi-process data parallel flag"}, + {name: "hybrid flag value", args: []string{"--disable-hybrid-kv-cache-manager=false"}, wantErr: "boolean flag"}, + {name: "hybrid split value", args: []string{"--disable-hybrid-kv-cache-manager", "false"}, wantErr: "boolean flag"}, + {name: "duplicate transfer config", args: []string{"--kv-transfer-config", "{}", "--kv-transfer-config={}"}, wantErr: "at most once"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := adapter.ValidateMPEnginePod(newVLLMMPEnginePod(tc.args...), newTypedVLLMMPBackend()) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("ValidateMPEnginePod: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateMPEnginePod error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + +func TestVLLMLMCacheMPKVTransferConfigRoles(t *testing.T) { + tests := []struct { + role cachev1alpha1.CacheBackendIntegrationRole + want string + }{ + {role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, want: kvRoleConsumer}, + {role: cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, want: kvRoleProducer}, + {role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, want: kvRoleBoth}, + {role: "", want: kvRoleBoth}, + } + for _, tc := range tests { + t.Run(string(tc.role), func(t *testing.T) { + raw, err := vllmMPKVTransferConfigJSON(tc.role, 6500) + if err != nil { + t.Fatalf("vllmMPKVTransferConfigJSON: %v", err) + } + var got vllmMPKVTransferConfig + if err := json.Unmarshal([]byte(raw), &got); err != nil { + t.Fatalf("unmarshal config %q: %v", raw, err) + } + if got.Connector != vllmLMCacheMPConnectorName || + got.ConnectorModule != vllmLMCacheMPConnectorModulePath || + got.Role != tc.want || + got.ExtraConfig.Host != "tcp://127.0.0.1" || + got.ExtraConfig.Port != "6500" { + t.Fatalf("config = %+v", got) + } + }) + } + if _, err := vllmMPKVTransferConfigJSON("future", 6500); err == nil { + t.Fatal("unknown role unexpectedly admitted") + } +} + +func TestVLLMLMCacheMPInjectsCommonServerAndExternalConnector(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + cache := newTypedVLLMMPBackend() + pod := newVLLMMPEnginePod("--model", "meta-llama/Meta-Llama-3-8B-Instruct", "--tensor-parallel-size=2") + pod.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "KEEP_ME", Value: "yes"}, + {Name: EnvLMCacheRemoteURL, Value: "lm://legacy:8200"}, + {Name: EnvLMCacheChunkSize, Value: "128"}, + {Name: EnvPythonHashSeed, Value: "random"}, + } + + if err := adapter.InjectEngineConfig(&pod.Spec, respBinding("redis.ns1.svc.cluster.local:6379"), cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + engine := &pod.Spec.Containers[0] + if !containsArg(engine.Args, vllmDisableHybridKVCacheArg) { + t.Fatalf("engine args missing %s: %v", vllmDisableHybridKVCacheArg, engine.Args) + } + values, malformed := argValues(engine.Args, defaultEngineKVTransferConfigArg) + if malformed || len(values) != 1 { + t.Fatalf("kv transfer config values=%v malformed=%v args=%v", values, malformed, engine.Args) + } + var config vllmMPKVTransferConfig + if err := json.Unmarshal([]byte(values[0]), &config); err != nil { + t.Fatalf("unmarshal injected config: %v", err) + } + if config.Connector != vllmLMCacheMPConnectorName || config.ConnectorModule != vllmLMCacheMPConnectorModulePath || config.Role != kvRoleBoth { + t.Fatalf("injected connector config = %+v", config) + } + if got, ok := lookupEnv(engine.Env, EnvPythonHashSeed); !ok || got != "0" { + t.Fatalf("%s = %q, %v", EnvPythonHashSeed, got, ok) + } + if got, ok := lookupEnv(engine.Env, EnvInferenceCacheFailOpen); !ok || got != "true" { + t.Fatalf("%s = %q, %v", EnvInferenceCacheFailOpen, got, ok) + } + if got, ok := lookupEnv(engine.Env, "KEEP_ME"); !ok || got != "yes" { + t.Fatalf("unrelated env was not preserved: %q, %v", got, ok) + } + for _, legacy := range []string{EnvLMCacheRemoteURL, EnvLMCacheChunkSize} { + if _, ok := lookupEnv(engine.Env, legacy); ok { + t.Fatalf("legacy env %s survived typed MP injection: %+v", legacy, engine.Env) + } + } + + server := findInitContainer(pod.Spec.InitContainers, lmCacheMPServerContainerName) + if server == nil { + t.Fatalf("common MP server missing: %+v", pod.Spec.InitContainers) + } + if server.Image != cache.Spec.LMCache.PodLocal.Server.Image || !containsArg(server.Args, "--l2-adapter") { + t.Fatalf("MP server = %+v", server) + } + if findVolume(pod.Spec.Volumes, lmCacheMPConfigVolumeName) != nil || hasMount(engine.VolumeMounts, lmCacheMPConfigVolumeName) { + t.Fatalf("vLLM must not receive SGLang YAML config volume: volumes=%+v mounts=%+v", pod.Spec.Volumes, engine.VolumeMounts) + } + if findVolume(pod.Spec.Volumes, lmCacheMPShmVolumeName) == nil || !hasMount(engine.VolumeMounts, lmCacheMPShmVolumeName) { + t.Fatalf("shared MP shm missing: volumes=%+v mounts=%+v", pod.Spec.Volumes, engine.VolumeMounts) + } + + want := pod.Spec.DeepCopy() + if err := adapter.InjectEngineConfig(&pod.Spec, respBinding("redis.ns1.svc.cluster.local:6379"), cache); err != nil { + t.Fatalf("second InjectEngineConfig: %v", err) + } + if !reflect.DeepEqual(&pod.Spec, want) { + t.Fatalf("typed vLLM MP injection is not idempotent\n got=%+v\nwant=%+v", pod.Spec, *want) + } +} + +func TestVLLMLMCacheMPInjectsRedisSecretReferencesIntoServerOnly(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + cache := newTypedVLLMMPBackend() + pod := newVLLMMPEnginePod("--model", "model") + binding := &backendadapter.Binding{ + Protocol: backendadapter.ProtocolRESP, + Endpoint: "redis.example:6379", + Redis: &backendadapter.RedisBinding{Authentication: &cachev1alpha1.RedisAuthenticationSpec{ + Username: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, + Key: "username", + }, + Password: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, + Key: "password", + }, + }}, + } + + if err := adapter.InjectEngineConfig(&pod.Spec, binding, cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + server := findInitContainer(pod.Spec.InitContainers, lmCacheMPServerContainerName) + if server == nil { + t.Fatal("common MP server missing") + } + assertSecretEnv := func(name, key string) { + t.Helper() + for i := range server.Env { + if server.Env[i].Name != name { + continue + } + ref := server.Env[i].ValueFrom + if ref == nil || ref.SecretKeyRef == nil || ref.SecretKeyRef.Name != "redis-auth" || ref.SecretKeyRef.Key != key { + t.Fatalf("%s = %+v, want redis-auth/%s SecretKeyRef", name, server.Env[i], key) + } + return + } + t.Fatalf("%s missing", name) + } + assertSecretEnv(lmCacheRESPUsernameEnv, "username") + assertSecretEnv(lmCacheRESPPasswordEnv, "password") + for _, env := range pod.Spec.Containers[0].Env { + if env.Name == lmCacheRESPUsernameEnv || env.Name == lmCacheRESPPasswordEnv { + t.Fatalf("Redis credential reference leaked into engine env: %+v", env) + } + } +} + +func TestVLLMLMCacheMPInjectionCollisionIsAtomic(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + cache := newTypedVLLMMPBackend() + pod := newVLLMMPEnginePod("--model", "model") + pod.Spec.InitContainers = []corev1.Container{{Name: lmCacheMPServerContainerName, Image: "user-owned"}} + want := pod.Spec.DeepCopy() + if err := adapter.InjectEngineConfig(&pod.Spec, (*backendadapter.Binding)(nil), cache); err == nil { + t.Fatal("foreign MP server collision unexpectedly admitted") + } + if !reflect.DeepEqual(&pod.Spec, want) { + t.Fatalf("failed injection mutated PodSpec\n got=%+v\nwant=%+v", pod.Spec, *want) + } +} diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index 41778bce..5e5d25e4 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -356,6 +356,101 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { t.Fatalf("typed native sidecar metrics label %s = %q, want %q", LabelLMCacheMPMetrics, got, LabelLMCacheMPMetricsEnabled) } + + // Typed vLLM PodLocal smoke: prove the dedicated MP adapter, external + // connector module path, deterministic hash seed, and common native sidecar + // survive real apiserver admission/defaulting. This remains a control-plane + // test; envtest has no kubelet or GPU runtime. + vllmMPCB := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "vllm-mp", Namespace: ns}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + "app": "vllm-mp-test", + }}, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + Port: 6555, + L1Capacity: resource.MustParse("1Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("2Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("3Gi")}, + }, + }}, + }, + }, + } + if err := mgr.GetClient().Create(ctx, vllmMPCB); err != nil { + t.Fatalf("create typed vLLM CacheBackend: %v", err) + } + vllmMPPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vllm-mp-engine", + Namespace: ns, + Labels: map[string]string{"app": "vllm-mp-test"}, + Annotations: map[string]string{ + "inferencecache.io/lmcache-connector-profile": "vllm-lmcache-mp-v1", + "inferencecache.io/lmcache-client-version": "0.5.3", + }, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: "vllm", Image: "vllm:connector-ready", Args: []string{"--model", "model-a", "--tensor-parallel-size=2"}, + }}}, + } + if err := mgr.GetClient().Create(ctx, vllmMPPod); err != nil { + t.Fatalf("create typed vLLM Pod: %v", err) + } + var gotVLLMMP corev1.Pod + if err := mgr.GetAPIReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: vllmMPPod.Name}, &gotVLLMMP); err != nil { + t.Fatalf("get typed vLLM Pod: %v", err) + } + var vllmMPServer *corev1.Container + for index := range gotVLLMMP.Spec.InitContainers { + if gotVLLMMP.Spec.InitContainers[index].Name == "lmcache-mp-server" { + vllmMPServer = &gotVLLMMP.Spec.InitContainers[index] + break + } + } + if vllmMPServer == nil || vllmMPServer.RestartPolicy == nil || *vllmMPServer.RestartPolicy != corev1.ContainerRestartPolicyAlways { + t.Fatalf("typed vLLM native sidecar = %+v", vllmMPServer) + } + vllmConfig := envtestArgValue(gotVLLMMP.Spec.Containers[0].Args, "--kv-transfer-config") + if !strings.Contains(vllmConfig, `"kv_connector":"LMCacheMPConnector"`) || + !strings.Contains(vllmConfig, `"kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector"`) || + !strings.Contains(vllmConfig, `"lmcache.mp.host":"tcp://127.0.0.1"`) { + t.Fatalf("typed vLLM kv-transfer-config = %q", vllmConfig) + } + if !envtestHasContainerEnvValue(&gotVLLMMP, "PYTHONHASHSEED", "0") { + t.Fatalf("typed vLLM Pod is missing PYTHONHASHSEED=0: %+v", gotVLLMMP.Spec.Containers[0].Env) + } +} + +func envtestArgValue(args []string, flag string) string { + for index, arg := range args { + if arg == flag && index+1 < len(args) { + return args[index+1] + } + if strings.HasPrefix(arg, flag+"=") { + return strings.TrimPrefix(arg, flag+"=") + } + } + return "" +} + +func envtestHasContainerEnvValue(pod *corev1.Pod, name, value string) bool { + if pod == nil || len(pod.Spec.Containers) == 0 { + return false + } + for _, entry := range pod.Spec.Containers[0].Env { + if entry.Name == name && entry.Value == value { + return true + } + } + return false } // mustHaveContainerEnv fails the test if the first container's env array diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index 2c5d998a..bdf71815 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -54,6 +54,7 @@ func newVLLMRegistry(configs ...builtinruntime.SubscriberConfig) *adapterruntime config = configs[0] } registry := adapterruntime.NewRegistry() + registry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(config)) registry.Register(builtinruntime.NewVLLMLMCacheAdapter(config)) return registry } @@ -322,26 +323,100 @@ func TestHandle_MatchAndInject(t *testing.T) { mustHaveArgFlag(t, mutated, "--kv-transfer-config") } -func TestHandle_TypedLMCacheDoesNotFallThroughToLegacyAdapter(t *testing.T) { - const ns = "engines" - cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) +func typedVLLMPodLocalBackend(name, namespace string, selector map[string]string) *cachev1alpha1.CacheBackend { + cb := readyCacheBackend(name, namespace, selector) + chunkSize := int32(256) cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ - Topology: cachev1alpha1.LMCacheTopologyPodLocal, + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + ChunkSizeTokens: &chunkSize, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + }, + }}, } cb.Spec.RemoteStorage = nil + cb.Status.Endpoint = "" + return cb +} + +func TestHandle_TypedVLLMMissingCapabilityDoesNotFallThroughToLegacyAdapter(t *testing.T) { + const ns = "engines" + cb := typedVLLMPodLocalBackend("primary", ns, map[string]string{"app": "vllm"}) h := newHandler(t, cb) pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) req := newRequest(t, pod, ns) resp := h.Handle(context.Background(), req) if !resp.Allowed { - t.Fatalf("typed MP pod must fail open while its adapter is pending: %+v", resp.Result) + t.Fatalf("typed MP pod with unverified connector must fail open: %+v", resp.Result) } if len(resp.Patches) != 0 { t.Fatalf("typed MP pod must not receive legacy injection; got %d patches", len(resp.Patches)) } - if resp.Result == nil || !strings.Contains(resp.Result.Message, "does not implement typed LMCache topology") { - t.Fatalf("response message = %v, want typed-topology adapter diagnostic", resp.Result) + if resp.Result == nil || !strings.Contains(resp.Result.Message, "engine connector capability is unverified") { + t.Fatalf("response message = %v, want connector-capability diagnostic", resp.Result) + } +} + +func TestHandle_TypedPodLocalVLLMUsesDedicatedMPAdapter(t *testing.T) { + const ns = "engines" + cb := typedVLLMPodLocalBackend("vllm-typed", ns, map[string]string{"app": "vllm-mp"}) + h := newHandler(t, cb) + pod := vllmEnginePod("engine-mp", map[string]string{"app": "vllm-mp"}) + pod.Annotations = map[string]string{ + adapterruntime.AnnotationLMCacheConnectorProfile: "vllm-lmcache-mp-v1", + adapterruntime.AnnotationLMCacheClientVersion: "0.5.3", + } + pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--tensor-parallel-size=2") + req := newRequest(t, pod, ns) + + resp := h.Handle(context.Background(), req) + if !resp.Allowed || len(resp.Patches) == 0 { + t.Fatalf("typed vLLM MP injection: Allowed=%v patches=%d result=%+v", resp.Allowed, len(resp.Patches), resp.Result) + } + mutated := applyPatches(t, req.Object.Raw, resp) + server := findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") + if server == nil || server.Image != cb.Spec.LMCache.PodLocal.Server.Image { + t.Fatalf("typed MP server = %+v", server) + } + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-worker") != nil { + t.Fatalf("typed vLLM wire fell through to legacy worker: %+v", mutated.Spec.InitContainers) + } + mustHaveArgFlag(t, mutated, "--disable-hybrid-kv-cache-manager") + config := testArgValue(mutated.Spec.Containers[0].Args, "--kv-transfer-config") + for _, want := range []string{ + `"kv_connector":"LMCacheMPConnector"`, + `"kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector"`, + `"kv_role":"kv_both"`, + `"lmcache.mp.host":"tcp://127.0.0.1"`, + `"lmcache.mp.port":"6500"`, + } { + if !strings.Contains(config, want) { + t.Fatalf("kv-transfer-config %q missing %q", config, want) + } + } + mustHaveEnv(t, mutated, testEnvPythonHashSeed, "0") + if got := mutated.Labels[LabelLMCacheMPMetrics]; got != LabelLMCacheMPMetricsEnabled { + t.Fatalf("label %s = %q", LabelLMCacheMPMetrics, got) + } + + incompatible := vllmEnginePod("engine-pp", map[string]string{"app": "vllm-mp"}) + incompatible.Annotations = pod.Annotations + incompatible.Spec.Containers[0].Args = append(incompatible.Spec.Containers[0].Args, "--pipeline-parallel-size=2") + incompatibleReq := newRequest(t, incompatible, ns) + incompatibleResp := h.Handle(context.Background(), incompatibleReq) + if !incompatibleResp.Allowed || len(incompatibleResp.Patches) != 0 { + t.Fatalf("unsupported PP must admit unchanged: Allowed=%v patches=%d result=%+v", + incompatibleResp.Allowed, len(incompatibleResp.Patches), incompatibleResp.Result) + } + if incompatibleResp.Result == nil || !strings.Contains(incompatibleResp.Result.Message, "pipeline parallel size 2") { + t.Fatalf("unsupported PP diagnostic = %+v", incompatibleResp.Result) } } @@ -2299,6 +2374,18 @@ func mustHaveArgPair(t *testing.T, pod *corev1.Pod, flag, value string) { t.Fatalf("arg pair %s %s missing; args = %v", flag, value, args) } +func testArgValue(args []string, flag string) string { + for index, arg := range args { + if arg == flag && index+1 < len(args) { + return args[index+1] + } + if strings.HasPrefix(arg, flag+"=") { + return strings.TrimPrefix(arg, flag+"=") + } + } + return "" +} + func mustHaveArgFlag(t *testing.T, pod *corev1.Pod, flag string) { t.Helper() for _, a := range pod.Spec.Containers[0].Args { diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index fa6e651c..5dd97875 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -293,9 +293,10 @@ func validateMPServerResourceRequirements(resources corev1.ResourceRequirements, return errs } -// rejectUnimplementedRedisBindingFeatures permits the Phase-2 SGLang MP auth -// path while keeping every unsupported LMCache 0.5.3 RESP feature explicit. -// That adapter supports username/password, but not TLS or logical database +// rejectUnimplementedRedisBindingFeatures permits authentication for typed +// PodLocal LMCache MP adapters while keeping every unsupported LMCache 0.5.3 +// RESP feature explicit. The common MP server renderer supports +// username/password for both SGLang and vLLM, but not TLS or logical database // selection. Managed Redis currently provisions the default user, so its // password may be configured but an ACL username may not. func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) field.ErrorList { @@ -307,11 +308,14 @@ func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) fie var errs field.ErrorList if redis.Authentication != nil { authPath := path.Child("authentication") - isSGLangMP := cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeSGLang && - cb.Spec.LMCache != nil && cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal - if !isSGLangMP { + isTypedPodLocalMP := cb.Spec.LMCache != nil && + cb.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && + cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal && + (cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeSGLang || + cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeVLLM) + if !isTypedPodLocalMP { errs = append(errs, field.Forbidden(authPath, - "Redis authentication is currently rendered only by the SGLang PodLocal LMCache MP adapter")) + "Redis authentication is currently rendered only by a typed PodLocal LMCache MP adapter")) } else { if redis.Authentication.Username != nil { errs = append(errs, validateRedisSecretKeySelector(*redis.Authentication.Username, authPath.Child("username"))...) diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index 8d938871..3c4ace41 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -297,7 +297,7 @@ func TestRejectUnimplementedRedisBindingFeatures(t *testing.T) { } } -func TestValidateRedisAuthenticationForSGLangPodLocal(t *testing.T) { +func TestValidateRedisAuthenticationForTypedPodLocal(t *testing.T) { newAuthBackend := func(ownership cachev1alpha1.CacheBackendRemoteStorageOwnership) *cachev1alpha1.CacheBackend { cb := validPodLocalMPBackend() cb.Name = "mp" @@ -376,12 +376,34 @@ func TestValidateRedisAuthenticationForSGLangPodLocal(t *testing.T) { } }) - t.Run("vLLM remains rejected until its MP adapter lands", func(t *testing.T) { + t.Run("vLLM PodLocal admitted", func(t *testing.T) { cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM + if errs := rejectUnimplementedRedisBindingFeatures(cb); len(errs) != 0 { + t.Fatalf("authentication errors = %v", errs) + } + if _, err := shippingValidator().ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("ValidateCreate: %v", err) + } + }) + + t.Run("legacy vLLM remains rejected", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM + cb.Spec.LMCache.Topology = "" + cb.Spec.LMCache.PodLocal = nil + errs := rejectUnimplementedRedisBindingFeatures(cb) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed PodLocal LMCache MP") { + t.Fatalf("errors = %v, want topology-scoped rejection", errs) + } + }) + + t.Run("non-LMCache type remains rejected", func(t *testing.T) { + cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) + cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache errs := rejectUnimplementedRedisBindingFeatures(cb) - if len(errs) != 1 || !strings.Contains(errs[0].Error(), "SGLang PodLocal") { - t.Fatalf("errors = %v, want runtime-scoped rejection", errs) + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed PodLocal LMCache MP") { + t.Fatalf("errors = %v, want cache-type-scoped rejection", errs) } }) } diff --git a/internal/webhook/v1alpha1/cachebackend_override_validation_test.go b/internal/webhook/v1alpha1/cachebackend_override_validation_test.go index 6b8f0d74..42839292 100644 --- a/internal/webhook/v1alpha1/cachebackend_override_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_override_validation_test.go @@ -40,6 +40,20 @@ func TestValidator_EngineOverrides_SuppressReservedArgRejected(t *testing.T) { "spec.integration.engineOverrides.suppressArgs[0]", "\"vllm\"") } +func TestValidator_TypedVLLMMPReservedSurfaceRejected(t *testing.T) { + v := &CacheBackendValidator{Registry: defaultShippingRegistry()} + cb := validPodLocalMPBackend() + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ + EngineOverrides: &cachev1alpha1.EngineInjectionOverrides{ + SuppressArgs: []string{"--disable-hybrid-kv-cache-manager"}, + }, + } + requireInvalidWithCause(t, v, cb, + "spec.integration.engineOverrides.suppressArgs[0]", + "--disable-hybrid-kv-cache-manager") +} + func TestValidator_EngineOverrides_OverrideReservedArgRejected(t *testing.T) { v := &CacheBackendValidator{Registry: stubRegistry()} // Two forms: bare flag and equals form. Both must trip the rule, since diff --git a/internal/webhook/v1alpha1/cachebackend_validator_test.go b/internal/webhook/v1alpha1/cachebackend_validator_test.go index 84f13b15..ebd70136 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator_test.go +++ b/internal/webhook/v1alpha1/cachebackend_validator_test.go @@ -49,6 +49,7 @@ func i32p(v int32) *int32 { return &v } func defaultShippingRegistry() *adapterruntime.Registry { registry := adapterruntime.NewRegistry() + registry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{})) registry.Register(builtinruntime.NewVLLMLMCacheAdapter(builtinruntime.SubscriberConfig{})) registry.Register(builtinruntime.NewSGLangLMCacheAdapter(builtinruntime.SubscriberConfig{})) registry.Register(builtinruntime.NewSGLangHiCacheAdapter(builtinruntime.SubscriberConfig{})) From 75e1d64408280a20862938200714805a8cbd19a8 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Tue, 11 Aug 2026 11:21:53 -0700 Subject: [PATCH 05/13] Harden LMCache PodLocal after GPU validation Apply the fixes discovered during live SGLang and vLLM A100 testing: account for shared-memory headroom, strengthen native ABI checks, support current vLLM events and SGLang metrics, and simplify runtime capability handling. Restrict LMCache integrations to the validated ReadWrite role and record Phase 3/4 evidence plus deferred investigations. Signed-off-by: Yue Sun --- api/v1alpha1/cachebackend_types.go | 20 +- .../inferencecache.io_cachebackends.yaml | 20 +- .../gpu-validation/kustomization.yaml | 44 + ...ackend-sglang-podlocal-external-redis.yaml | 2 +- ...achebackend-sglang-podlocal-host-only.yaml | 8 +- ...backend-sglang-podlocal-managed-redis.yaml | 4 +- config/samples/cachebackend-sglang.yaml | 4 +- ...ebackend-vllm-podlocal-external-redis.yaml | 2 +- .../cachebackend-vllm-podlocal-host-only.yaml | 10 +- ...hebackend-vllm-podlocal-managed-redis.yaml | 4 +- docs/design/cachebackend-api.md | 50 +- .../lmcache-multiprocess-migration-roadmap.md | 1125 ++++++++--------- .../scripts/default_install_smoke.sh | 8 +- .../builtin/runtime/lmcache_mp_renderer.go | 40 +- .../runtime/lmcache_mp_renderer_test.go | 54 +- .../adapters/builtin/runtime/lmcachecheck.go | 29 +- .../runtime/lmcachecheck_script_test.go | 48 +- .../builtin/runtime/sglang_hicache.go | 2 +- .../builtin/runtime/sglang_hicache_test.go | 1 + .../builtin/runtime/sglang_lmcache.go | 50 +- .../builtin/runtime/sglang_lmcache_test.go | 40 +- .../builtin/runtime/vllm_lmcache_mp.go | 18 +- .../builtin/runtime/vllm_lmcache_mp_test.go | 8 - .../controller/cachebackend_kernelcheck.go | 12 +- .../cachebackend_lmcache_mp_status.go | 4 +- internal/subscriber/events.go | 102 +- internal/subscriber/events_test.go | 83 ++ .../webhook/pod/envtest_integration_test.go | 8 - internal/webhook/pod/podinjector.go | 6 - internal/webhook/pod/podinjector_test.go | 26 +- .../cachebackend_integration_validation.go | 25 +- ...achebackend_integration_validation_test.go | 63 +- .../cachebackend_lmcache_mp_validation.go | 20 +- ...cachebackend_lmcache_mp_validation_test.go | 18 +- .../v1alpha1/cachebackend_validator.go | 2 +- .../v1alpha1/cachebackend_validator_test.go | 11 +- pkg/adapters/runtime/adapter.go | 40 - pkg/adapters/runtime/adapter_test.go | 52 - test/fixtures/sglang-lmcache/Dockerfile | 1 - 39 files changed, 1101 insertions(+), 963 deletions(-) create mode 100644 config/overlays/gpu-validation/kustomization.yaml diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index 02a015fa..b27681d2 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -209,8 +209,9 @@ type LMCachePodLocalServerSpec struct { // +kubebuilder:validation:Maximum=65535 Port int32 `json:"port"` - // L1Capacity is the server's host-memory cache capacity. Container memory - // requests and limits must leave positive headroom above this value. + // L1Capacity is the server's host-memory cache capacity. The renderer sizes + // /dev/shm to this value plus 1Gi; container memory requests and limits must + // each cover that complete budget. // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" L1Capacity resource.Quantity `json:"l1Capacity"` @@ -219,8 +220,8 @@ type LMCachePodLocalServerSpec struct { MaxWorkers int32 `json:"maxWorkers"` // Resources are applied to the injected MP server container. Admission - // requires positive CPU and memory requests plus a memory limit that leaves - // headroom above l1Capacity. + // requires a positive CPU request and requires both the memory request and + // memory limit to cover l1Capacity plus 1Gi of /dev/shm headroom. Resources corev1.ResourceRequirements `json:"resources"` } @@ -652,12 +653,11 @@ type CacheBackendIntegrationSpec struct { // ReadOnly / WriteOnly are specialised producer/consumer roles operators // opt into explicitly. // - // Engine support is per-adapter: vLLM maps the role onto its LMCache - // connector's kv_role (ReadOnly→kv_consumer, WriteOnly→kv_producer, - // ReadWrite→kv_both). The SGLang LMCache integration has no kv_role split - // (--enable-lmcache always both stores and retrieves), so a (sglang, - // LMCache) backend supports only ReadWrite — admission rejects ReadOnly / - // WriteOnly there rather than silently ignoring them. + // Support is backend-specific. LMCache currently supports only ReadWrite: + // SGLang has no directional role split, and the validated LMCache 0.5.3 + // vLLM MP connector does not enforce kv_consumer / kv_producer. Admission + // rejects ReadOnly / WriteOnly for every LMCache backend rather than expose + // directionality the data plane does not honor. // +optional // +kubebuilder:default=ReadWrite Role CacheBackendIntegrationRole `json:"role,omitempty"` diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 96473cde..437b86ee 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -527,12 +527,11 @@ spec: ReadOnly / WriteOnly are specialised producer/consumer roles operators opt into explicitly. - Engine support is per-adapter: vLLM maps the role onto its LMCache - connector's kv_role (ReadOnly→kv_consumer, WriteOnly→kv_producer, - ReadWrite→kv_both). The SGLang LMCache integration has no kv_role split - (--enable-lmcache always both stores and retrieves), so a (sglang, - LMCache) backend supports only ReadWrite — admission rejects ReadOnly / - WriteOnly there rather than silently ignoring them. + Support is backend-specific. LMCache currently supports only ReadWrite: + SGLang has no directional role split, and the validated LMCache 0.5.3 + vLLM MP connector does not enforce kv_consumer / kv_producer. Admission + rejects ReadOnly / WriteOnly for every LMCache backend rather than expose + directionality the data plane does not honor. enum: - ReadOnly - WriteOnly @@ -1672,8 +1671,9 @@ spec: - type: integer - type: string description: |- - L1Capacity is the server's host-memory cache capacity. Container memory - requests and limits must leave positive headroom above this value. + L1Capacity is the server's host-memory cache capacity. The renderer sizes + /dev/shm to this value plus 1Gi; container memory requests and limits must + each cover that complete budget. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -1695,8 +1695,8 @@ spec: resources: description: |- Resources are applied to the injected MP server container. Admission - requires positive CPU and memory requests plus a memory limit that leaves - headroom above l1Capacity. + requires a positive CPU request and requires both the memory request and + memory limit to cover l1Capacity plus 1Gi of /dev/shm headroom. properties: claims: description: |- diff --git a/config/overlays/gpu-validation/kustomization.yaml b/config/overlays/gpu-validation/kustomization.yaml new file mode 100644 index 00000000..4e5be615 --- /dev/null +++ b/config/overlays/gpu-validation/kustomization.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Shared-cluster GPU validation overlay. Keep the shipped default install +# cluster-wide for its CRD/controller surfaces, but ask the API server to send +# Pod CREATE admission requests to the inference-cache mutator only from the +# dedicated test namespace. CacheBackend/CachePolicy/CacheTenant admission is +# intentionally unchanged. +resources: +- ../../default + +images: +- name: ghcr.io/cachebox-project/inference-cache-controller + newName: sjc.ocir.io/idqj093njucb/inference-cache-controller + digest: sha256:6dcab2344027ef8ac3db2ab22352cdaa77d80202ec11df49dddeeefe08095b18 +- name: ghcr.io/cachebox-project/inference-cache-server + newName: sjc.ocir.io/idqj093njucb/inference-cache-server + digest: sha256:f735a5e69280411995f1e15d1a19b40c462e450bcf7ea0012a0c5520fb778d46 + +patches: +- target: + group: apps + version: v1 + kind: Deployment + name: inference-cache-controller-manager + patch: | + - op: replace + path: /spec/template/spec/containers/0/args/1 + value: --lmcache-server-image=lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + - op: add + path: /spec/template/spec/containers/0/args/- + value: --kvevent-subscriber-image=sjc.ocir.io/idqj093njucb/inference-cache-subscriber@sha256:2fdaa611642a0f2c48b6c7a7257ff28d75030e4bf6df3cebb589319f2e48e504 +- target: + group: admissionregistration.k8s.io + version: v1 + kind: MutatingWebhookConfiguration + name: inference-cache-mutating-webhook-configuration + patch: | + - op: add + path: /webhooks/0/namespaceSelector + value: + matchLabels: + kubernetes.io/metadata.name: inference-cache-gpu-test diff --git a/config/samples/cachebackend-sglang-podlocal-external-redis.yaml b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml index 080c9417..1d935792 100644 --- a/config/samples/cachebackend-sglang-podlocal-external-redis.yaml +++ b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml @@ -24,7 +24,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 diff --git a/config/samples/cachebackend-sglang-podlocal-host-only.yaml b/config/samples/cachebackend-sglang-podlocal-host-only.yaml index 88b31e6b..da72ca6c 100644 --- a/config/samples/cachebackend-sglang-podlocal-host-only.yaml +++ b/config/samples/cachebackend-sglang-podlocal-host-only.yaml @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # Typed SGLang PodLocal LMCache MP with no remote tier. The inference-owner Pod -# image must contain lmcache==0.5.3 and declare the connector capability shown -# in docs/design/lmcache-multiprocess-migration-roadmap.md. Its launch args must -# explicitly set --page-size to a divisor of chunkSizeTokens. +# image must contain a compatible LMCache client/API; normal engine startup +# fails before serving when it does not. No capability annotation is required. +# Launch args must explicitly set --page-size to a divisor of chunkSizeTokens. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -23,7 +23,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 diff --git a/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml index 4cbfaf4e..029d8660 100644 --- a/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml +++ b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml @@ -21,7 +21,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 @@ -36,7 +36,7 @@ spec: provider: Redis ownership: Managed redis: - image: redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 + image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: requests: cpu: 500m diff --git a/config/samples/cachebackend-sglang.yaml b/config/samples/cachebackend-sglang.yaml index 31c18f2f..2d350ae3 100644 --- a/config/samples/cachebackend-sglang.yaml +++ b/config/samples/cachebackend-sglang.yaml @@ -55,8 +55,8 @@ spec: deploymentKind: Deployment replicas: 1 integration: - # Only ReadWrite is supported for (sglang, LMCache): SGLang's --enable-lmcache - # has no producer/consumer split, so admission rejects ReadOnly / WriteOnly. + # LMCache currently admits only ReadWrite for every engine; directional + # roles require a connector that has been validated to enforce them. role: ReadWrite engineSelector: matchLabels: diff --git a/config/samples/cachebackend-vllm-podlocal-external-redis.yaml b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml index a84bc17a..00fe5808 100644 --- a/config/samples/cachebackend-vllm-podlocal-external-redis.yaml +++ b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml @@ -24,7 +24,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 diff --git a/config/samples/cachebackend-vllm-podlocal-host-only.yaml b/config/samples/cachebackend-vllm-podlocal-host-only.yaml index 38f98b8d..f831119c 100644 --- a/config/samples/cachebackend-vllm-podlocal-host-only.yaml +++ b/config/samples/cachebackend-vllm-podlocal-host-only.yaml @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -# Typed vLLM PodLocal LMCache MP with no remote tier. Matching engine Pods must -# declare connector profile vllm-lmcache-mp-v1 and LMCache client version 0.5.3. -# CacheBackend injects the MP server sidecar and connector JSON but never -# changes the engine image. +# Typed vLLM PodLocal LMCache MP with no remote tier. CacheBackend injects the +# MP server sidecar and connector JSON but never changes the engine image. The +# engine's normal startup fails before serving if its image lacks a compatible +# LMCache client/API; no capability annotation is required. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -23,7 +23,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 diff --git a/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml index 89041e13..e42f1e64 100644 --- a/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml +++ b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml @@ -21,7 +21,7 @@ spec: chunkSizeTokens: 256 podLocal: server: - image: lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 port: 5555 l1Capacity: 4Gi maxWorkers: 4 @@ -36,7 +36,7 @@ spec: provider: Redis ownership: Managed redis: - image: redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 + image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: requests: cpu: 500m diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 7b0d7b1d..77e3f000 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -117,7 +117,7 @@ and [`config/samples/cachebackend-mooncake.yaml`](../../config/samples/cacheback | `autoscaling.maxReplicas` | integer | Upper bound for HPA replica count. Required when `autoscaling` is set. Minimum `1`. Cross-field validation: `minReplicas <= maxReplicas`. | | `autoscaling.targetCPUUtilizationPercent` | integer | Target average per-pod CPU utilization for the HPA. Defaults to `80` when unset. Range `[1, 100]`. | | `integration.mode` | enum | Which cache tiers the engine is wired for: `Offload` (default) or `EventsOnly`. `Offload` is full participation — cache-aware routing (tier-1) plus the KV-offload connector (tier-2). It may remain host-only, connect to externally owned remote storage, or provision a provider workload when `remoteStorage.ownership` is `Managed`. `EventsOnly` wires routing only: the kvevent-subscriber sidecar is injected when the controller runs with `--kvevent-subscriber-image` set and `observation.modelID` is present; otherwise the append is skipped fail-open. No KV connector or backend server is created. See [Events-only mode](#events-only-mode-specintegrationmode--eventsonly). | -| `integration.role` | enum | Engine participation mode: `ReadOnly`, `WriteOnly`, or `ReadWrite`. Defaults to `ReadWrite`. | +| `integration.role` | enum | Engine participation mode: `ReadOnly`, `WriteOnly`, or `ReadWrite`. Defaults to `ReadWrite`. LMCache currently admits only `ReadWrite`; directional roles remain reserved for a connector that demonstrably enforces them. | | `integration.failOpen` | boolean | Default `true`. When `true`, engine pods fall back to local prefill on cache unreachability — the cache is an optimization, never a serving dependency. Setting it to `false` is an advanced opt-in to fail-closed serving (the cache becomes a serving dependency); the controller surfaces this as a Warning Kubernetes Event on the owning `CacheBackend`. **Pair-specific exception — `(sglang, LMCache)`:** SGLang has no cacheless code path while `--enable-lmcache` is on, so its co-scheduled MP worker is a *serving prerequisite* (a worker that never starts wedges the engine), not a remote dependency that degrades to local prefill. `failOpen` is still honored at the tier that can actually be "unavailable" — the shared L2 (the worker comes up L1-only when Redis is unreachable). This is a documented, accepted boundary; see the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md) and [SGLang engine support](#sglang-engine-support). | | `integration.engineOverrides` | object | Optional engine-injection overrides applied to the args/env the pod-mutating webhook would otherwise inject into the engine container. See [Engine-injection overrides](#engine-injection-overrides-specintegrationengineoverrides). | | `engineSelector.matchLabels` | map | Equality-based label selector matched against engine **pod** labels (the pod template's `metadata.labels`, not Deployment, DaemonSet, or any other workload-level labels). Every key/value here must appear on the pod for it to match. `matchExpressions` is intentionally not exposed in v1alpha1 — the surface is `matchLabels` only. | @@ -198,25 +198,22 @@ and must be sized on that workload instead. The typed shape `spec.runtime: VLLM`, `spec.type: LMCache`, and `spec.lmCache.topology: PodLocal` selects a dedicated MP adapter; it does not reuse the legacy `LMCacheConnectorV1` / `lm://` path. The engine image remains -owned by the inference runtime. To make that image's capability explicit, each -matching Pod must declare: - -```yaml -metadata: - annotations: - inferencecache.io/lmcache-connector-profile: vllm-lmcache-mp-v1 - inferencecache.io/lmcache-client-version: "0.5.3" -``` +owned by the inference runtime. No connector-profile annotation or image +allowlist is required: this CacheBackend shape is the only enablement switch. +The webhook validates Pod-visible topology and arguments, then injects the MP +wire. The engine's normal initialization loads the connector and fails before +serving if its image does not contain a compatible LMCache client/API; +admission does not pull, execute, or otherwise introspect the engine image. The webhook injects a digest-pinned `lmcache-mp-server` native sidecar and adds the following vLLM launch contract: - `--kv-transfer-config` selects `LMCacheMPConnector` through `lmcache.integration.vllm.lmcache_mp_connector`, points it at - `tcp://127.0.0.1:`, and maps the integration role to - `kv_consumer`, `kv_producer`, or `kv_both`; -- `--disable-hybrid-kv-cache-manager` is required by the initial pinned - profile; + `tcp://127.0.0.1:`, and sets `kv_role: kv_both` for the + only currently admitted LMCache role, `ReadWrite`; +- `--disable-hybrid-kv-cache-manager` is required by the initial validated + integration; - `PYTHONHASHSEED=0` stabilizes vLLM's cross-process hash chain; - `INFERENCECACHE_FAIL_OPEN` mirrors the API setting, although runtime-native failure behavior still requires GPU validation. @@ -231,6 +228,15 @@ vLLM image/version, KV reuse, TP determinism, and failure recovery remain Phase 4 runtime gates. Canonical examples are the three `config/samples/cachebackend-vllm-podlocal-*.yaml` files. +For both typed vLLM and SGLang PodLocal adapters, `l1Capacity` is the usable L1 +target, not the complete container budget. The common renderer creates a +memory-backed `/dev/shm` with `sizeLimit: l1Capacity + 1Gi`; admission requires +both the MP-server memory request and memory limit to be at least that value. If +the engine already mounts `/dev/shm`, the adapter reuses it only when it is a +memory-backed `emptyDir` with a `sizeLimit` at least as large as that budget. +This keeps scheduling/cgroup accounting aligned with the tmpfs and leaves room +for LMCache metadata and shared-memory allocator overhead. + ### SGLang engine support SGLang supports two peer cache integrations: @@ -296,7 +302,7 @@ The old lm:// `LMCACHE_REMOTE_URL` / serde / chunk-size / local-CPU env is Deliberately **not** injected for SGLang (a real engine difference, not an omission): `VLLM_USE_V1` (a vLLM-internal codepath with no SGLang analogue) and `PYTHONHASHSEED` (vLLM pins it to stabilise its builtin-`hash()`-seeded block-hash chain across TP workers; SGLang derives its prefix hash with `hashlib.sha256` over the token-id bytes, independent of `PYTHONHASHSEED`). -**`spec.integration.role` support.** vLLM maps the role onto its LMCache connector's `kv_role` (ReadOnly→`kv_consumer`, WriteOnly→`kv_producer`, ReadWrite→`kv_both`). SGLang's `--enable-lmcache` integration has **no `kv_role` split** — it always both stores and retrieves — so a `(sglang, LMCache)` backend supports only `ReadWrite` (the default). Admission **rejects** `ReadOnly` / `WriteOnly` for SGLang (`rejectUnsupportedSGLangRole`) rather than silently treating them as ReadWrite; the rule lifts if SGLang's LMCache integration gains a producer/consumer split. +**`spec.integration.role` support.** Every LMCache backend currently supports only `ReadWrite` (the default), and admission rejects `ReadOnly` / `WriteOnly` through `rejectUnsupportedLMCacheRole`. SGLang's `--enable-lmcache` path has no role split. vLLM can render `kv_consumer` / `kv_producer`, but live GPU validation found that LMCache 0.5.3 still stored in consumer mode and retrieved in producer mode. Directional roles remain in the generic API for other backends and a future validated LMCache connector, but inference-cache does not claim semantics the selected data plane cannot enforce. **Reserved set** (`internal/adapters/builtin/runtime`): `ReservedArgs()` = `--enable-lmcache`, `--lmcache-config-file`; `ReservedEnv()` = `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. In MP mode the old lm:// `LMCACHE_REMOTE_URL` is neither injected nor reserved. `VLLM_USE_V1` / `PYTHONHASHSEED` are not reserved because they are never injected. @@ -438,15 +444,19 @@ wire-protocol skew above, in its **local-kernel** variant. Because `import lmcache.c_ops` is overridden to a fallback shim on load failure (so it always succeeds and cannot be used as a health check), the control plane detects this at **deploy time** with an injected `lmcache-kernel-check` init -container that force-loads the native extension from disk in the engine's own -image. It reports onto the CacheBackend `EngineKernelsHealthy` condition (see +container that force-loads LMCache `c_ops` from disk and imports the vLLM core +native extension shipped by the engine image (`vllm._C_stable_libtorch` in +current stable-ABI builds, falling back to legacy `vllm._C`). Checking both +LMCache and vLLM matters because their `libcudart` dependencies can differ. It +reports onto the CacheBackend +`EngineKernelsHealthy` condition (see [Conditions](#conditions)) and is configured per-CacheBackend via the `inferencecache.io/lmcache-kernel-check` annotation: | Annotation value | Behavior | |---|---| | `auto` (default / unset) | Inject in report-only mode **only** when the engine container requests a GPU (the kernels are GPU-only; a CPU build legitimately has none). | -| `report-only` | Always inject; a `c_ops` load failure makes the detector exit 0, so it does not block the engine pod (best-effort fail-open — see the residual cases in [Boundaries](#boundaries-what-the-check-does-and-does-not-prove)). The condition surfaces the result. | +| `report-only` | Always inject; a native LMCache/vLLM load failure makes the detector exit 0, so it does not block the engine pod (best-effort fail-open — see the residual cases in [Boundaries](#boundaries-what-the-check-does-and-does-not-prove)). The condition surfaces the result. | | `strict` | Always inject; on failure the engine pod stays in `Init` and never serves (fail-closed), and the managed CacheBackend `Ready` is downgraded with reason `EngineKernelDegraded`. | | `off` | Never inject. | @@ -479,13 +489,13 @@ Extending the check to SGLang is a follow-up. kernel launch. That residual is caught only at runtime. - **Strict-mode GPU cost:** a pod stuck in `Init` (failing the check in strict mode) still holds its `nvidia.com/gpu` reservation while serving nothing. - Reclaim it by fixing the engine image's lmcache/CUDA alignment or switching + Reclaim it by fixing the engine image's vLLM/LMCache/CUDA alignment or switching the annotation to `report-only`. - The check runs `import torch` (the native extension links libtorch), adding a few seconds to GPU engine-pod startup. The engine imports torch anyway. - **Report-only fail-open is best-effort.** The init container runs the engine image's own `python3`; in report-only mode the detector always exits 0, so a - `c_ops` failure never blocks the pod. The init container declares small CPU/ + native-extension failure never blocks the pod. The init container declares small CPU/ memory requests and no limits — the most broadly-compatible shape, but note that *no* resource shape is fail-open under every namespace policy: a `ResourceQuota`/`LimitRange` that requires per-container requests rejects a diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 53818263..dbfd69a1 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -1,6 +1,6 @@ # Design Roadmap: LMCache Multiprocess Migration -Status: **engineering-validated; Phase 0 complete (2026-08-09)** · Scope: +Status: **Phases 0–4 complete (2026-08-11)** · Scope: deprecate and remove this project's LMCache in-process data plane, converge vLLM and SGLang on LMCache multiprocess (MP) mode, and model Pod-local and node-local MP server placement without conflating either with @@ -21,7 +21,7 @@ replacement exists. Use a strangler migration: 1. freeze the IP path and lock the target API; 2. introduce an MP-only `CacheBackend` surface; 3. extract and harden the common MP server infrastructure using SGLang; -4. add vLLM MP in explicit opt-in mode; +4. add vLLM MP behind the typed PodLocal `CacheBackend` shape; 5. migrate all repository-owned samples, tests, and manifests; 6. confirm that the Phase 0 no-consumer assumption still holds, then remove the IP adapter, `lm://` provider, and legacy lifecycle code; and @@ -46,14 +46,13 @@ commitment under the Phase 0 finding. - The initial remote L3 scope is RESP/Redis. MP + Mooncake Store, S3, NIXL, and other adapters are separate follow-ups; they must not be implied by accepting an inert provider declaration. -- Native sidecars require a supported Kubernetes version. The exact minimum - Kubernetes version is locked in Phase 0 and enforced/documented before the MP - path becomes the default. +- Native sidecars require Kubernetes 1.29 or newer with `SidecarContainers`; + Kubernetes 1.33 or newer is recommended. - The inference-workload owner supplies and pins the engine image; CacheBackend never rewrites it. CacheBackend pins the cache components it injects or - manages, and the selected runtime adapter validates the connector/server - compatibility profile. Mixed MP client/server versions are not assumed - wire-compatible. + manages. The selected adapter renders the connector contract; normal engine + initialization is the authoritative compatibility check. Mixed MP + client/server versions are not assumed wire-compatible. ## Terminology and tier model @@ -101,21 +100,21 @@ code lands. | D7 | Connector endpoints are not published in the generic `status.endpoint`. | PodLocal uses loopback; NodeLocal is node-dependent. Only remote L3 has a globally meaningful provider endpoint. | | D8 | Unsupported combinations are rejected at admission. | An accepted but inert cache field commonly produces silent zero-hit behavior. | | D9 | Fail-open is rendered into runtime-native behavior and tested. | A custom environment variable without a known consumer is not an enforceable serving contract. | -| D10 | Provider restart recovery is capability-specific. | Legacy `lm://` socket recovery, MP server recovery, and remote L3 recovery have different semantics and blast radii. | -| D11 | Each supported vLLM profile explicitly identifies its MP connector implementation; the initial reference profile uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial profile uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future profile may validate a different implementation explicitly. | -| D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. Runtime adapters must declare and validate the connector capabilities they require without turning a tested engine image into an API allowlist or mutation default. | +| D10 | Component lifecycle ownership is capability-specific. | The Pod-local MP process is kubelet-owned while remote L3 is independently managed; connector re-registration after an MP-process restart is a post-migration enhancement, not an MVP contract. | +| D11 | Each supported vLLM integration explicitly identifies its MP connector implementation; the initial reference baseline uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial adapter uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future adapter revision may validate a different implementation explicitly. | +| D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. The selected adapter renders its engine-specific connector contract, while normal engine initialization is the authoritative compatibility check; tested images are neither an admission allowlist nor a mutation default. | ## Current state | Area | Current behavior | Gap to target | |---|---|---| -| SGLang engine wire | Implicit MP; injects a Pod-local native sidecar, config file, loopback endpoint, and shared `/dev/shm`; the sidecar image defaults to the engine image. | Renderer is SGLang-private; cache-component ownership is coupled to the workload image; legacy ZMQ-only server entry point; incomplete worker health/recovery and parallelism coverage. | +| SGLang engine wire | Implicit MP; injects a Pod-local native sidecar, config file, loopback endpoint, and shared `/dev/shm`; the sidecar image defaults to the engine image. | Renderer is SGLang-private; cache-component ownership is coupled to the workload image; legacy ZMQ-only server entry point; incomplete worker health and parallelism coverage. | | vLLM engine wire | `LMCacheConnectorV1` with optional host CPU, `lm://`, or `mooncakestore://`. | No `LMCacheMPConnector`; IP is still the only vLLM LMCache implementation. | | CR API | MP mode is inferred from runtime. `hostMemory`, `workerImage`, `workerPort`, and `remoteSerde` are flat sibling fields. | No explicit MP topology; mode-specific fields can be accepted and ignored. | | Remote storage | `Redis`, `LMCacheServer`, and `Mooncake` share one provider abstraction. | `LMCacheServer` is a legacy connector service, not a general MP L3; Mooncake needs a different MP binding shape. | | Lifecycle | Every managed provider participates in the cache-server restart cascade. | Redis L3 restarts can roll engine fleets even though the engine connects to a local MP server. | | Status | Provider readiness and engine-container crash loops are observed. | Native-sidecar health and node coverage are not represented; `status.endpoint` is ambiguous. | -| Tests | Strong Go unit coverage; SGLang single-GPU evidence; sample admission checks. | No default-install engine-Pod injection smoke; no vLLM MP; no automated GPU fault/parallelism matrix. | +| Tests | Strong Go unit coverage; SGLang single-GPU evidence; sample admission checks. | No default-install engine-Pod injection smoke; no vLLM MP; no automated GPU parallelism/Redis matrix. | ### Connector ownership @@ -128,18 +127,21 @@ and both engines require code from LMCache: | SGLang | LMCache-specific `LMCRadixCache`, `--enable-lmcache`, and `--lmcache-config-file` integration. It is not selected through vLLM's generic connector registry. | SGLang imports `LMCacheMPConnector` and related adapters from `lmcache.integration.sglang`; LMCache also supplies the MP server. | Therefore neither engine image is self-sufficient merely because it exposes an -LMCache flag or connector class. A runtime adapter must verify that the engine -image contains the required LMCache client package/API, then CacheBackend injects -and manages a compatible MP server without replacing that engine image. +LMCache flag or connector class. A runtime adapter renders the required +engine-specific wire, then CacheBackend injects and manages a compatible MP +server without replacing the engine image. The engine's normal initialization +is the authoritative check that its image actually contains the required +LMCache client package/API. Source support is not the same as image support. The upstream vLLM Dockerfile defaults `INSTALL_KV_CONNECTORS=false`, so its connector source may be present while the `lmcache` runtime dependency is absent. SGLang likewise documents installing `lmcache` separately; its integration raises an error when that import is unavailable. CacheBackend cannot fix a missing Python package by injecting -flags or a server sidecar. A supported runtime profile must therefore establish -that the workload image already contains the required LMCache client and expose -enough version/capability metadata for the adapter to select a compatible server. +flags or a server sidecar. The inference owner must therefore choose a +compatible image, but does not declare a second CacheBackend capability switch: +connector import or API incompatibility fails during normal engine startup +before the Pod serves. ## Target architecture @@ -166,8 +168,8 @@ Properties: - engine and server share a Pod network namespace and `/dev/shm`; - no `hostNetwork` is required; - cross-Pod sharing requires a configured remote L3; -- lifecycle is coupled, but mid-flight server restart behavior must still be - defined and tested. +- lifecycle is Pod-coupled; mid-flight MP-server restart and connector + re-registration are explicitly deferred to post-migration improvements. ### NodeLocal @@ -341,7 +343,7 @@ Condition and Event contract: | Signal | Semantics | |---|---| -| `ConnectorReady` | `Unknown/ConnectorCapabilityUnverified` until the runtime declaration and Pod shape are verified; `False` for a known incompatibility or unhealthy required MP server; `True` only when the selected engines are covered by healthy MP servers. | +| `ConnectorReady` | `Unknown/ConnectorInjectionUnverified` until the current CacheBackend generation has been injected into every selected Pod; `False` for an unhealthy required MP server or engine Pod; `True` only when selected engines are Ready and covered by healthy MP servers. Runtime package/API incompatibility is surfaced by normal engine startup/readiness, not a capability declaration. | | `RemoteStorageReady` | Omitted when no L3 is configured; otherwise `Unknown/RemoteStoragePending`, `False/RemoteStorageUnavailable`, or `True/RemoteStorageReady`, independently of connector health. | | `LegacyInProcessDeprecated` | Conditional compatibility signal only if a legacy consumer appears before physical removal: condition `True` plus a Warning Event with the same reason and a migration instruction. It is never set for typed MP objects. | @@ -356,8 +358,8 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 0 | Design freeze, consumer audit, version/Kubernetes baseline | none | complete | | 1 | MP-only API and admission/status contracts | Phase 0 | complete | | 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | complete | -| 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | in progress — non-GPU baseline complete; GPU matrix pending | -| 4 | vLLM PodLocal MP, host-only and Redis | Phase 3 | in progress — non-GPU control-plane baseline complete; engine/GPU matrix pending | +| 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | complete | +| 4 | vLLM PodLocal MP | Phase 3 | complete | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | | 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 finding | | 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | not started | @@ -365,537 +367,338 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. ## Phase 0 — design freeze and compatibility baseline -### Goal +- **Status:** Complete (2026-08-09) +- **Depends on:** None -Stop the target from moving while implementation begins and determine whether -the alpha removal carve-out is safe to use. +### Objective + +Freeze the MP-only target and confirm that an in-place `v1alpha1` cleanup is +safe. + +### Scope + +Includes architecture decisions, repository/consumer inventory, migration +policy, and validation baselines. It does not change the data plane. ### Deliverables -- [x] Complete the engineering review and obtain project-owner approval for - D1–D12. -- [x] Inventory all repository manifests and confirm the external/installed - population of `CacheBackend` objects using: - - vLLM IP host-only; - - `remoteStorage.provider: LMCacheServer`; - - existing IP-only `remoteStorage.provider: Mooncake`; - - SGLang MP flat worker fields. -- [x] Decide the API migration strategy: - - **selected:** in-place `v1alpha1` cleanup because there are zero external - consumers and zero installed legacy objects; or - - served compatibility period if that fact changes before removal. -- [x] Pin and record the Phase 3/4 target tuple for each engine: - - reference engine image/digest, used only as a validation fixture; - - CacheBackend-injected server image/digest; - - LMCache version; - - CUDA/runtime version; - - Kubernetes version; - - target model and parallelism validation modes. -- [x] Freeze new feature work on `LMCacheConnectorV1`, `lm://`, and the managed - legacy LMCache server. -- [x] Resolve whether top-level managed-provider fields (`replicas`, - `autoscaling`, `deploymentKind`, `template`) move below `remoteStorage` in - the same alpha API cleanup. They must not accidentally configure an MP - server with a different lifecycle. - - **Selected:** retain them only as legacy provider-workload inputs during the - repository migration, then relocate any still-needed fields below a typed - `remoteStorage` managed-workload block in Phase 7. They never configure the - PodLocal/NodeLocal MP server; that lifecycle lives exclusively under - `lmCache.podLocal.server` or `lmCache.nodeLocal.server`. - -The locked Phase 3/4 validation targets are reference environments, not engine -image requirements or admission allowlists: - -| Component/profile | Reference validation environment | Ownership | Required validation | -|---|---|---|---| -| LMCache MP server | `lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13`; LMCache 0.5.3; linux/amd64; CUDA 13.0.1 | CacheBackend-injected and digest-pinned | Client/server compatibility with both reference runtimes, probes, restart, and recovery | -| vLLM connector profile | `lmcache/vllm-openai@sha256:dca0afdda6ad1bb02e63619d366fcd18975b334d7274739ed6f2025035865781`; LMCache 0.5.3; explicit `lmcache.integration.vllm.lmcache_mp_connector`; linux/amd64; CUDA 13.0.1 | Inference-owner image; test fixture only | Llama 3.1 8B; TP=1/2, plus TP=4 before common multi-GPU recommendation | -| SGLang connector profile | linux/amd64 from `lmsysorg/sglang@sha256:1c64fde976bdf0d56474a30bccbcfc19667e5b3ab34c826a534c9d6aaca41212` (`v0.5.13.post1-cu130`) with exactly `lmcache==0.5.3`; CUDA 13.0 | Inference-owner image; derived test fixture only | `--enable-lmcache`/config-file capability; Llama 3 8B; TP=1/2 | -| Redis L3 | `redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2` (`7.4.10-alpine`) | CacheBackend-managed when not external | Both engines: cross-Pod reuse, credentials/TLS, outage, and recovery | - -Kubernetes 1.29 with `SidecarContainers` enabled is the minimum; 1.33 or newer -is recommended. All four published registry digests resolved on 2026-08-09, and -the SGLang and Redis indexes contain linux/amd64 manifests. The SGLang derived -test-fixture digest is produced in Phase 3. The vLLM reference digest pins its -contents, but its upstream release build did not constrain the vLLM package -version; Phase 4 preflight must record `vllm.__version__` and reject that -reference profile if incompatible. None of these engine-image choices authorize -CacheBackend to mutate a workload's image. +- [x] Approve decisions D1–D12 and freeze new work on `LMCacheConnectorV1`, + `lm://`, and the legacy managed LMCache server. +- [x] Confirm there are no external `CacheBackend` users and no installed + legacy objects that must remain readable. +- [x] Select an in-place alpha cleanup; activate a compatibility window only if + a legacy consumer appears before removal. +- [x] Set Kubernetes 1.29 plus `SidecarContainers` as the minimum; recommend + 1.33 or newer. +- [x] Keep engine images runtime-owned. Validation records exact versions and + digests, but does not create an image allowlist or universal CUDA tuple. +- [x] Keep top-level managed-provider workload fields legacy-only until Phase 7; + they never configure PodLocal or NodeLocal MP servers. ### Validation -- [x] Repository search and sample inventory are attached to the design PR. -- [x] Current Go tests and sample verification pass before behavior changes. -- [x] Every unsupported or unverified combination has an explicit disposition. - -The repository manifest inventory is: +- [x] Baseline commit `083e916` passed `go test ./...`, `make verify-samples` + (21 admitted, 2 intentional skips), naming checks, and internal-reference + checks. +- [x] Repository inventory completed: -| Migration class | Objects | Disposition | +| Legacy class | Count | Disposition | |---|---:|---| -| vLLM IP + `LMCacheServer` | 13 | Migrate repository samples and references after vLLM MP passes Phase 4. | -| vLLM IP + existing engine-side Mooncake provider | 1 | Do not reinterpret as MP `mooncake_store`; migrate explicitly or defer until that adapter is supported. | -| vLLM IP host-only | 1 | Intentionally invalid test fixture; update with the admission tests. | -| SGLang MP with flat fields + Redis | 1 | Move to the typed PodLocal block in Phase 5. | -| SGLang MP with flat fields, host-only | 1 | Move to the typed PodLocal block in Phase 5. | -| EventsOnly, no LMCache data plane | 1 | No data-plane migration. | - -Direct repository consumers also include the reference-stack manifest and Helm -values, default-install checks, C2/C6 scripts, and their fixtures. The -`SGLangHiCache` sample is outside this LMCache migration. The project owner -confirmed that there are no external `CacheBackend` consumers and no installed -legacy objects requiring migration, so live-cluster inventory is not required. - -Baseline validation at commit `083e916` passed `go test ./...` and -`make verify-samples` (21 admitted, 2 explicit skips, 0 failures). Naming and -internal-reference checks also passed. These are API/data-plane baselines, not -GPU or kubelet-native-sidecar evidence; those gates remain in Phases 2–4. +| vLLM IP + `LMCacheServer` | 13 | Migrate repository-owned samples after Phase 4. | +| vLLM IP + engine-side Mooncake | 1 | Migrate explicitly; do not reinterpret as MP Mooncake Store. | +| vLLM IP host-only fixture | 1 | Replace with MP admission coverage. | +| SGLang flat MP fields | 2 | Convert to typed PodLocal in Phase 5. | +| EventsOnly | 1 | No LMCache data-plane migration. | + +Initial runtime validation uses LMCache client/server 0.5.3. SGLang requires +TP=1; vLLM requires TP=1 and TP=2 on one node. Redis evidence is supplemental, +not a Phase 3/4 exit gate. ### Exit criteria -- The project owner confirmed that no external or installed legacy consumer can - be broken by the chosen in-place alpha cleanup. -- The immutable candidate images, software versions, Kubernetes baseline, and - Phase 3/4 validation matrix are published. -- The target CR and migration policy are approved. +- [x] Migration policy and target API approved. +- [x] Consumer audit supports the in-place alpha cleanup. +- [x] Runtime and Kubernetes validation baselines recorded. ## Phase 1 — MP-only API, admission, and status contract -### Goal - -Add the final MP shape before changing the data plane, while retaining only the -minimum compatibility surface needed to migrate existing IP objects. - -### API work - -- [x] Add typed MP-only `spec.lmCache` configuration. Because MP is the only - canonical LMCache data plane, do not add a redundant `multiprocess` - nesting level. -- [x] Add the `PodLocal` topology and its typed server configuration. -- [x] Design the `NodeLocal` block now, but reject it until Phase 8 is - implemented. Do not accept inert NodeLocal objects. -- [x] Make host-only MP explicit by allowing `remoteStorage` to be absent. -- [x] Remove `LMCacheServer` from the canonical MP provider matrix while - retaining the legacy enum for topology-less repository objects until - Phase 7. -- [x] Define structured remote-provider bindings that can grow beyond - `Binding{Protocol, Endpoint}` to carry credentials, TLS, and adapter - parameters without stringly typed engine overrides. Typed Redis - credential/TLS/database fields initially reject unsupported use, so they - cannot be accepted and silently ignored; Phase 2 enables Secret-backed - SGLang RESP authentication while retaining explicit TLS/database - rejections for the pinned adapter. -- [x] Define connector and remote-storage status separately. -- [x] Define how a workload declares or exposes its engine and LMCache-client - capability/version without allowing CacheBackend to rewrite the engine - image or making the admission webhook pull arbitrary registry content. -- [x] Define migration/deprecation conditions and Events. - -### Compatibility work - -- [x] Mark the old flat fields as legacy inputs: - - `lmCache.hostMemory`; - - `lmCache.workerImage`; - - `lmCache.workerPort`; - - `lmCache.remoteSerde`. -- [x] Prevent new objects from mixing old flat fields with the typed MP - topology. -- [x] Preserve the current runtime-derived behavior for existing objects until - Phase 6; do not silently switch existing vLLM IP objects to MP. -- [x] Define old-to-new field mappings in the migration table below. - -The runtime owner, not CacheBackend, selects and pins the inference image. Its -validated Pod template declares -`inferencecache.io/lmcache-connector-profile` and -`inferencecache.io/lmcache-client-version`. The image build pipeline must probe -the required connector import/entry point and record the package version before -publishing that declaration. At Pod admission, the selected typed MP adapter -compares the declaration with its required profile and validates observable -engine args/resources; admission never pulls the image or contacts a registry. -Successful mutation also stamps the CacheBackend generation rendered into the -immutable Pod. `ConnectorReady` treats an older generation as unverified until -the inference owner recreates or rolls that Pod; a CacheBackend spec update -cannot retroactively rewrite a running engine Pod. -An absent/mismatched declaration, an adapter that has not implemented the typed -MP contract, or an unclassifiable engine topology is admitted fail-open without -cache mutation and with an actionable diagnostic. CacheBackend never rewrites -the engine image. The concrete profile probes and supported version tuples land -with the Phase 2 renderer and Phase 3/4 runtime adapters. - -| Legacy field | Typed MP disposition | +- **Status:** Complete +- **Depends on:** Phase 0 + +### Objective + +Introduce the final typed MP API and reject configurations that would otherwise +be accepted but ignored. + +### Scope + +Includes CRD shape, defaulting, validation, compatibility rules, and status +types. It does not change the runtime data plane. + +### Deliverables + +- [x] Add typed `spec.lmCache` with `PodLocal`; design but reject `NodeLocal` + until Phase 8. +- [x] Allow host-only MP by omitting `remoteStorage`. +- [x] Keep `LMCacheServer` only for topology-less legacy objects until Phase 7. +- [x] Add structured Redis bindings; support Secret-backed authentication and + reject unsupported TLS/database fields. +- [x] Separate connector status from remote-storage status. +- [x] Make `CacheBackend.spec` the only enablement switch. Engine Pods require + no connector-profile/version annotations, and admission never pulls or + executes their images. +- [x] Preserve legacy behavior temporarily, but reject mixing typed topology + with `hostMemory`, `workerImage`, `workerPort`, or `remoteSerde`. +- [x] Prevent typed MP objects from falling through to a legacy runtime adapter. + +Migration mapping: + +| Legacy input | Typed disposition | |---|---| -| `lmCache.hostMemory.capacity` | Copy to `lmCache.podLocal.server.l1Capacity`; separately choose explicit server resources with memory headroom. | -| `lmCache.workerImage` | Copy only after pinning it by digest to `lmCache.podLocal.server.image`. | -| `lmCache.workerPort` | Copy to `lmCache.podLocal.server.port` after collision validation. | -| `lmCache.remoteSerde` | No automatic mapping; remove it unless a future typed L3 adapter exposes and validates equivalent semantics. | -| `lmCache.chunkSizeTokens` | Remains `lmCache.chunkSizeTokens`; it is common connector configuration, not topology nesting. | -| `remoteStorage.provider: LMCacheServer` | No automatic provider mapping; explicitly select host-only or a supported L3 such as Redis. | - -`lmCache.podLocal.server.maxWorkers` and `resources` are new required choices; -legacy objects do not contain enough information to derive production-safe -values. - -### Admission invariants - -- [x] Exactly one topology-specific block matches `topology`. -- [x] `PodLocal` rejects `nodeLocal`; `NodeLocal` rejects `podLocal`. -- [x] MP rejects `remoteStorage.provider: LMCacheServer`. -- [x] SGLang and vLLM reject remote providers their selected MP adapter cannot - render. -- [x] L1 capacity is positive and has a schedulable memory budget with explicit - headroom. -- [x] Ports are valid and do not collide with known operator-owned MP/event - ports. -- [x] Version-sensitive or unvalidated parallelism combinations fail loudly at - the boundary where the topology is observable: CR admission for declared - fields, and engine-Pod admission for engine args/resources. A combination - that cannot be classified is not silently injected. -- [x] `remoteSerde` cannot be supplied to MP. -- [x] EventsOnly cannot carry MP or remote-storage configuration that will not - be used. - -### Tests - -- [x] CRD schema/defaulting unit tests. -- [x] Validating webhook table tests for every topology/provider combination. -- [x] Envtest CREATE/UPDATE compatibility tests. -- [x] Round-trip/deep-copy tests for all new typed fields. -- [x] Status serialization tests. Condition transitions land with the Phase 2 - status writer because Phase 1 intentionally changes no data plane. +| `hostMemory.capacity` | `podLocal.server.l1Capacity`; choose resources separately. | +| `workerImage` | `podLocal.server.image`, pinned by digest. | +| `workerPort` | `podLocal.server.port`, after collision validation. | +| `chunkSizeTokens` | Remains at `lmCache.chunkSizeTokens`. | +| `remoteSerde` | No automatic mapping. | +| `remoteStorage.provider: LMCacheServer` | Explicitly choose host-only or a supported L3. | + +### Validation + +- [x] Validate topology/block consistency, supported providers, positive L1 + budget, memory headroom, ports, parallel arguments, and EventsOnly rules. +- [x] CRD/defaulting, webhook table, deepcopy/round-trip, status serialization, + and envtest CREATE/UPDATE tests pass. +- [x] Pod-visible incompatible or unclassifiable shapes fail open without a + partial mutation and produce an actionable diagnostic. ### Exit criteria -- New PodLocal MP objects admit with or without Redis. -- Every impossible combination is rejected at admission. -- Existing IP objects still reconcile unchanged during the compatibility - period. -- Every MP field has an identified renderer/status consumer, and typed MP - objects cannot fall through to a legacy runtime adapter while those Phase 2-4 - consumers are landing. +- [x] PodLocal admits with or without Redis. +- [x] Unsupported combinations are rejected rather than ignored. +- [x] Existing legacy objects remain reconcilable until removal. +- [x] Every new field has a renderer or status consumer. ## Phase 2 — engine-neutral PodLocal MP server renderer -### Goal - -Turn the existing SGLang-specific spike into shared infrastructure before vLLM -depends on it. - -### Refactoring work - -- [x] Introduce an engine-neutral internal MP server configuration model. -- [x] Extract native-sidecar, config-volume, `/dev/shm`, resources, probes, - security context, and L3 adapter rendering from - `sglang_lmcache_wire.go`. -- [x] Keep engine launch surfaces separate: - - SGLang config file and `--enable-lmcache`; - - vLLM `LMCacheMPConnector` JSON and deterministic hash settings. -- [x] Preserve atomic and idempotent Pod mutation. -- [x] Preserve reserved-name and mount-collision checks. - -### Runtime work - -- [x] Replace `python3 -m lmcache.v1.multiprocess.server` with the supported - `lmcache server` entry point for the pinned LMCache version. -- [x] Add HTTP startup, readiness, and liveness probes. -- [x] Expose/scrape Prometheus metrics. -- [x] Add typed worker-pool sizing (`maxWorkers` initially; split GPU/CPU pools - when required by the pinned version and test matrix). -- [x] Add explicit CPU, memory, and optional ephemeral-storage resources. -- [x] Stop defaulting the MP sidecar to the engine image. Select the - CacheBackend-owned standalone server image by digest without modifying the - engine container image. -- [x] Let each runtime adapter declare its required engine-side connector - capability and supported client/server profiles. Surface an explicit - warning/condition when the observed runtime cannot be verified. -- [x] Render the Redis features supported by the pinned RESP adapter through - structured binding: Secret-backed authentication is wired on both ends; - unsupported TLS/database fields remain rejected instead of being silently - ignored. - -The exact LMCache 0.5.3 source constrains these runtime capability boundaries: - -- its `resp` adapter supports username/password (rendered from `SecretKeyRef`; - managed Redis supports the default user plus password), but it does not - support TLS or logical database selection. Admission therefore keeps - TLS/database rejected instead of accepting inert configuration. A future - validated Valkey adapter/image profile is required before those fields can be - used; -- `lmcache server` disables the separate Prometheus listener because its - FastAPI HTTP frontend already registers `/metrics` on `--http-port` (8080). - The renderer exposes the named `lmcache-http` port, successful typed PodLocal - injection stamps a stable metrics label, and the optional observability - overlay ships a cross-namespace `PodMonitor` for that label and route. - -### Lifecycle work - -- [x] Add MP native-sidecar health observation from - `status.initContainerStatuses`. -- [x] Stop treating every managed provider restart as an engine-restart event. -- [x] Introduce capability-specific restart behavior for: - - MP server restart; - - Redis L3 restart; - - legacy `lm://` restart during the compatibility window. -- [x] Define the Phase 2 recovery boundary: report a native-sidecar outage - through `ConnectorReady` and rely on kubelet liveness restart. Phase 3 - GPU-validates whether the pinned SGLang connector re-registers without an - engine restart. - -### Tests - -- [x] Renderer unit tests independent of SGLang. -- [x] Golden Pod tests for resources, probes, security, mounts, and L3 args. -- [x] Re-injection/idempotence tests. -- [x] Foreign volume/container collision tests. -- [x] Kubernetes 1.31 envtest admission smoke for native-sidecar fields. -- [x] Connector/remote-storage status condition-transition tests. -- [x] Pinned LMCache 0.5.3 standalone-image smoke: the exact Phase 0 digest - starts `lmcache server` through its CPU fallback, `/healthcheck` returns - healthy, and `/metrics` returns Prometheus text on HTTP port 8080. +- **Status:** Complete +- **Depends on:** Phase 1 + +### Objective + +Provide one engine-neutral PodLocal MP server renderer shared by SGLang and +vLLM. + +### Scope + +Includes sidecar rendering, shared memory, resources, health, metrics, Redis +binding, and lifecycle/status ownership. Engine launch arguments remain in each +runtime adapter. + +### Deliverables + +- [x] Extract a common renderer for the native sidecar, config volume, + memory-backed `/dev/shm`, resources, probes, security context, and L3 + arguments. +- [x] Use the supported `lmcache server` entry point and a digest-pinned + standalone image; never replace the engine image. +- [x] Preserve atomic/idempotent mutation and collision checks. +- [x] Keep SGLang config/flags separate from vLLM connector JSON and hash + settings. +- [x] Add typed `maxWorkers`, CPU/memory/ephemeral-storage resources, and the + `l1Capacity + 1Gi` shared-memory budget. +- [x] Expose `/healthcheck` and FastAPI `/metrics` on the server HTTP port. +- [x] Wire Redis username/password through `SecretKeyRef`; continue rejecting + TLS/database settings unsupported by LMCache 0.5.3 RESP. +- [x] Make Kubelet own the native sidecar and Redis lifecycle independent from + engine rollout. +- [x] Report server coverage through `ConnectorReady` and remote L3 through + `RemoteStorageReady`. + +### Validation + +- [x] Renderer golden, idempotence, collision, resource, security, mount, L3, + and status-transition tests pass. +- [x] Kubernetes envtest accepts the native-sidecar schema. +- [x] The pinned LMCache 0.5.3 standalone image starts through CPU fallback; + `/healthcheck` is healthy and `/metrics` returns Prometheus text on port + 8080. ### Exit criteria -- SGLang uses the common renderer with no data-plane regression. -- The server exposes a real health endpoint and metrics. -- The controller can distinguish MP server failure from engine failure and - remote-L3 failure. -- No Redis restart causes an unconditional engine-fleet rollout. +- [x] SGLang uses the common renderer without regression. +- [x] Health and metrics endpoints are real and observable. +- [x] Connector, engine, and remote-L3 failures are distinguishable. +- [x] Redis lifecycle does not automatically roll engine Pods. ## Phase 3 — SGLang PodLocal MP production baseline -### Goal - -Use the already working SGLang path to validate the common MP server under -parallelism and failure before adding vLLM. - -Current state: the non-GPU control-plane and connector compatibility baseline -is complete. No Phase 3 GPU data-path or runtime-failure result is claimed yet. - -### Completed non-GPU validation - -- [x] Add typed host-only, managed-Redis, and external-Redis SGLang PodLocal - samples; all three pass CRD defaulting and admission. -- [x] Add the connector-ready SGLang fixture Dockerfile under - `test/fixtures/sglang-lmcache`, based on the pinned SGLang digest with - exactly `lmcache==0.5.3`. -- [x] Build the fixture locally for linux/amd64 and verify SGLang - `0.5.13.post1`, CUDA 13.0.1, LMCache 0.5.3, `LMCacheMPConnector` import, - and CLI parsing for LMCache, explicit page size, and TP=2 flags. This is - compatibility preflight only; the image has not run inference. -- [x] Require an explicit SGLang `--page-size` and reject missing, malformed, - duplicate, non-positive, and declared chunk-incompatible values before - rendering the MP wire. LMCache retains its authoritative runtime check - against the effective page size. -- [x] Verify the Pod webhook renders the common MP server atomically and admits - an incompatible engine unchanged with an actionable fail-open diagnostic. -- [x] Persist a typed SGLang Pod through an envtest kube-apiserver/etcd and - verify the native-sidecar schema/defaulting surface. -- [x] Install the controller and webhooks in a Kubernetes 1.32 kind cluster, - create a matching SGLang Pod through the live mutating webhook, read the - persisted injected Pod back from the API server, and verify image, - restart policy, probes, resources, MP arguments, engine arguments, - annotations, labels, and shared mounts. The Pod was deliberately left - unscheduled, so no engine or sidecar container ran. -- [x] Pass the repository regression gates: full Go tests, focused envtest, - sample verification (24 pass, 2 intentional skips), default-install - smoke, Go vet, Prometheus rules, docs sync, REUSE, and DCO. - -### GPU/runtime functional scope - -- [ ] SGLang + PodLocal + no remote L3. Typed sample and admission wire are - complete; GPU KV execution is pending. -- [ ] SGLang + PodLocal + managed Redis development profile. Typed sample, - managed workload rendering, and admission pass; GPU KV execution is - pending. -- [ ] SGLang + PodLocal + external Redis production profile. Typed sample and - credential binding admission pass; GPU KV execution is pending. -- [x] ReadWrite role; reject unsupported role splits. -- [ ] Pinned SGLang/LMCache/CUDA image tuple. Local build and compatibility - preflight pass; registry digest and GPU execution are pending. - -### Correctness work - -- [ ] Exercise LMCache's runtime chunk-size check against the effective SGLang - page size on the pinned GPU tuple. The explicit-value admission guard and - its edge-case tests are complete. -- [ ] Validate TP=1 and TP=2 at minimum. -- [ ] Prove store → engine-GPU flush → retrieve from MP L1. -- [ ] Prove cross-Pod store/retrieve through Redis with fresh engine and MP L1. -- [ ] Verify event hash-domain separation and routing behavior remain correct. -- [ ] Verify cache eviction cannot create an indefinitely silent stale-affinity - signal without an observable metric/condition. - -### Failure work - -- [ ] Kill the MP server process and verify the selected recovery policy. -- [ ] Hang the MP server and verify liveness recovery. -- [ ] Restart the Pod-local native sidecar without replacing the engine process. -- [ ] Stop, restart, and replace Redis. -- [ ] Exhaust or nearly exhaust MP L1 memory and verify bounded eviction rather - than node OOM. -- [ ] Verify fail-open behavior with runtime-native evidence. - -### Operability work - -- [ ] `ConnectorReady` reflects MP server health. Condition-transition tests - pass; live SGLang failure evidence is pending. -- [ ] `RemoteStorageReady` reflects Redis independently. Condition-transition - tests pass; live Redis failure evidence is pending. -- [ ] Metrics prove lookup/store/retrieve/hit behavior. Metrics discovery and - Pod labeling pass; real KV traffic evidence is pending. -- [ ] Logs identify engine Pod, backend, model, MP instance, and L3 adapter - without exposing credentials. -- [x] Default-install smoke creates a matching SGLang engine Pod through the - live webhook and inspects the actual injected wire. +- **Status:** Complete +- **Depends on:** Phase 2 + +### Objective + +Establish a production-credible SGLang PodLocal baseline on the shared renderer. + +### Scope + +Includes one TP=1 engine Pod on one node, host-only MP as the required path, +resource pressure, events, metrics, and status. Managed Redis is supplemental. +SGLang TP>1, multi-node execution, model-specific features, external L3, and +sidecar restart recovery are outside this phase. + +### Deliverables + +- [x] Add typed host-only and optional-Redis samples plus a pinned validation + fixture with SGLang and LMCache compatibility checks. +- [x] Require an explicit valid `--page-size` compatible with + `chunkSizeTokens`; reject malformed or ambiguous values atomically. +- [x] Support ReadWrite and reject unsupported SGLang ReadOnly/WriteOnly roles. +- [x] Verify the complete mutation through unit tests, envtest, a live + Kubernetes 1.32 webhook, default-install smoke, and repository regression + gates. +- [x] Decode and index real SGLang KV events in a routing domain distinct from + vLLM; bound stale affinity through removal, TTL, and capacity eviction. + +### Validation + +Live validation ran on 2026-08-10/11 in SJC dev: + +| Item | Evidence | +|---|---| +| Environment | Kubernetes 1.31.1; one A100-SXM4-80GB; driver 550.163.01; `Qwen/Qwen2.5-0.5B-Instruct` | +| Engine | `lmsysorg/sglang@sha256:920df39109c60429b0a23eaacfd2786fcf1595c12f3ca4fc6e153b2abe34865f` (`0.5.13.post1-cu129`) | +| LMCache | Client wheel 0.5.3 CUDA 12.9, test-only runtime-owner overlay; standalone sidecar `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b` | +| Host-only TP=1 | Stored and retrieved 1,280 tokens from MP L1 after flushing the engine GPU cache; real KV events reached `KVEventsObserved`. | +| Chunk/page compatibility | LMCache chunk 256 and SGLang page 64 initialized and served successfully. | +| L1 pressure | A 64 MiB L1 reached 86%, evicted to 68.75%, proved partial-prefix eviction, and had no OOM or restart. | +| Managed Redis | Replacement engine Pod with fresh GPU/L1 retrieved 768 tokens from retained L2 data. Redis loss/recovery changed only `RemoteStorageReady`; the engine did not restart. | +| Metrics/status | Connector and remote-storage conditions transitioned independently; real SGLang metrics and subscriber recovery were observed. | + +The runtime-owner overlay is validation scaffolding, not authority for +CacheBackend to modify an engine image. Samples now use fully qualified +`docker.io/...` references because CRI-O rejected short registry names. + +- [x] Host-only TP=1 store → GPU flush → MP L1 retrieve. +- [x] Bounded L1 eviction without OOM or container restart. +- [x] Supplemental cross-Pod managed-Redis retrieval and Redis recovery without + engine rollout. +- [x] Live KV events, routing/index updates, metrics, and independent connector/ + remote-storage status. +- [x] Redis credentials remain `SecretKeyRef` values and are not copied into + process arguments. ### Exit criteria -- All required SGLang GPU and failure tests pass on the pinned tuple. -- A restarted or unhealthy MP server cannot leave a Ready engine silently - caching nothing indefinitely. -- Redis loss degrades to the documented local behavior without unnecessary - engine rollout. -- The SGLang sample and design document match the implementation. +- [x] Required SGLang TP=1 host-only GPU correctness tests pass and exact + validation images are recorded. +- [x] Resource pressure, events, metrics, status, and Redis lifecycle evidence + pass. +- [x] Samples and design match the implementation. ## Phase 4 — vLLM PodLocal MP -### Goal +- **Status:** Complete +- **Depends on:** Phase 3 -Provide the complete replacement for the current vLLM IP path before any IP -consumer is forced to migrate. +### Objective -Current state: the dedicated adapter and non-GPU control-plane baseline are -complete. No vLLM process has loaded the connector and no GPU KV operation or -runtime failure/recovery result is claimed yet. Phase 3's remaining GPU work -does not block this independent adapter work, but both phases retain their GPU -exit gates. +Provide the PodLocal MP replacement for the legacy vLLM IP path. -### Completed non-GPU validation +### Scope -- [x] Inspect the exact LMCache 0.5.3 source and its operator golden config. - The required vLLM wire is the external `LMCacheMPConnector` module - `lmcache.integration.vllm.lmcache_mp_connector`, with - `lmcache.mp.host` and `lmcache.mp.port` in - `kv_connector_extra_config`. This is source-level contract evidence, not - proof that the pinned vLLM reference image contains a compatible vLLM. -- [x] Add a dedicated typed vLLM MP adapter and register it before the legacy - vLLM adapter. Registry tests prove typed PodLocal objects select MP while - topology-less objects retain the legacy adapter during the compatibility - window. -- [x] Require the engine owner to declare connector profile - `vllm-lmcache-mp-v1` and LMCache client version `0.5.3`; a missing or - mismatched declaration admits the Pod unchanged with a fail-open - diagnostic rather than guessing from its image name. -- [x] Render the exact connector module, loopback address, configured port, and - role mapping (`ReadOnly`/`WriteOnly`/`ReadWrite` to - `kv_consumer`/`kv_producer`/`kv_both`). -- [x] Reuse the engine-neutral PodLocal renderer for the digest-pinned native - sidecar, bounded `/dev/shm`, probes, resources, and optional RESP L2. - Redis username/password remain `SecretKeyRef` values on the MP server and - are never copied into engine arguments or environment variables. -- [x] Remove legacy `LMCACHE_REMOTE_URL`, serde, chunk, and local-CPU env from - the typed wire; vLLM consumes connector JSON instead of SGLang's client - YAML volume. -- [x] Inject `PYTHONHASHSEED=0`, reserve the connector/hybrid arguments and - correctness-critical env, and preserve atomic/idempotent mutation. -- [x] Accept positive TP declarations, reject PP or DP greater than one, - external/multi-process DP flags, malformed/duplicate parallel flags, and - inject `--disable-hybrid-kv-cache-manager` for the initial profile. -- [x] Add typed host-only, managed-Redis, and external-Redis vLLM samples. All - three pass real envtest API-server admission; the external profile's - Secret-backed authentication is accepted while TLS/database remain - explicitly unsupported by LMCache 0.5.3. -- [x] Persist a typed vLLM Pod through envtest kube-apiserver/etcd and verify - the external connector module, loopback MP address, deterministic hash - seed, and common native-sidecar schema. -- [x] Install the controller and webhooks in a Kubernetes 1.32 kind cluster, - create a connector-declared vLLM Pod through the live mutating webhook, - read the persisted object back from etcd, and verify the exact MP wire. - An impossible node selector kept the Pod unscheduled, so neither the - engine nor MP server ran and no engine image was pulled. -- [x] Pass the non-GPU regression gates: full Go tests, focused envtest, - sample verification (27 pass, 2 intentional skips), default-install - smoke, Go vet, Prometheus rules, naming/internal-reference checks, docs - sync, and REUSE lint. - -The pinned 29 GiB vLLM reference image was deliberately not pulled or built for -this non-GPU baseline. Recording its exact `vllm.__version__`, importing the -connector inside that image, and exercising the engine CLI remain the first -runtime preflight before GPU testing. - -### Engine wire - -- [x] Add a dedicated vLLM MP adapter; do not mutate the legacy adapter in place. -- [x] Render the connector module path, role, loopback host, and configured MP - port in `LMCacheMPConnector` JSON. -- [ ] Add MQ timeout/heartbeat fields only when a pinned public configuration - surface exists. LMCache 0.5.3 currently supplies internal defaults, so - the API and renderer do not invent unsupported knobs. -- [ ] Prove runtime-native load-failure recompute/fail-open behavior. -- [x] Inject `PYTHONHASHSEED=0`; cross-process hash determinism still requires - the GPU/runtime test below. -- [x] Reserve only correctness-critical args/env owned by the typed adapter. -- [ ] Complete hybrid/parallelism classification for the pinned tuple. The - deterministic Pod-visible PP/DP restrictions and hybrid-manager guard are - implemented; MLA/model-specific behavior remains runtime validation. - -### Functional scope - -- [ ] vLLM + PodLocal + no remote L3. -- [ ] vLLM + PodLocal + managed Redis development profile. -- [ ] vLLM + PodLocal + external Redis production profile. -- [x] ReadOnly, WriteOnly, and ReadWrite connector JSON rendering. Runtime KV - behavior remains part of the GPU matrix. -- [ ] TP=1 and TP=2; TP=4 before recommending the topology for common multi-GPU - production workloads. -- [ ] Multi-server, DP + multi-server, and unsupported PP/MLA combinations are - rejected, not silently attempted. PP>1, DP>1, and external DP flags are - already rejected from Pod-visible arguments; multi-server and MLA - classification remain pending. - -### Correctness and failure tests - -- [ ] Store → GPU flush → retrieve from Pod-local MP L1. -- [ ] Cross-Pod retrieve through Redis. -- [ ] TP hash determinism with a negative test showing the zero-hit failure when - `PYTHONHASHSEED` is not pinned. -- [ ] MP server crash/restart/re-registration. -- [ ] MP server hang/liveness recovery. -- [ ] Redis loss and recovery. -- [ ] Engine rollout while MP/L3 data remains available as designed. -- [ ] Version-skew negative test. +Includes one vLLM engine Pod on one node at TP=1 and TP=2. Host-only MP is the +required path; managed Redis is supplemental. Multi-node/distributed execution, +MLA/model-specific behavior, external L3, and sidecar restart recovery are +outside this phase. No particular vLLM/CUDA image tuple is a production gate. + +### Deliverables + +- [x] Add a dedicated typed vLLM MP adapter using + `lmcache.integration.vllm.lmcache_mp_connector`; retain the legacy adapter + only for topology-less objects until removal. +- [x] Keep `CacheBackend.spec` as the only LMCache MP switch; no engine Pod + capability annotation or image inspection is required. +- [x] Render loopback host/port, `kv_both`, `PYTHONHASHSEED=0`, and + `--disable-hybrid-kv-cache-manager`; remove IP-only env/config. +- [x] Reuse the common sidecar, shared-memory, resources, probes, status, and + optional Redis binding. +- [x] Reject PP>1, DP>1, external/multi-process DP, and malformed or duplicate + parallel arguments; accept single-node TP. +- [x] Add host-only/Redis samples and verify unit, envtest, live Kubernetes 1.32 + admission, default-install smoke, and repository regression gates. +- [x] Repair the subscriber's observed vLLM tagged-map event decoding while + retaining SGLang/legacy tuple compatibility. +- [x] Enforce `/dev/shm = l1Capacity + 1Gi` and verify both LMCache and vLLM + native extensions through the existing kernel-check mechanism. + +### Validation + +Live validation ran on 2026-08-10/11 in SJC dev: + +| Item | Evidence | +|---|---| +| Environment | Kubernetes 1.31.1; A100-SXM4-80GB; driver 550.163.01; CUDA 12.9 | +| Engine | `us-sanjose-1.ocir.io/idqj093njucb/vllm-openai@sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (vLLM 0.25.1) | +| LMCache | Client wheel 0.5.3 CUDA 12.9, test-only runtime-owner overlay; standalone sidecar `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b` | +| Host-only TP=1 | Stored and retrieved 1,024 external tokens after clearing only vLLM's local prefix cache. | +| Host-only TP=2 | Both ranks registered and retrieved correctly on one node. | +| Events/status | Repaired tagged-map decoding passed live traffic; backend reached `Ready=True/KVEventsObserved`. | +| Shared memory | A 4 GiB L1 used a 5 GiB `/dev/shm`; accelerator serialization stayed enabled without pickle fallback. | +| Native ABI | LMCache `c_ops` and vLLM `_C_stable_libtorch` loaded; legacy `_C` fallback and missing-extension failures have regression tests. | +| Managed Redis | Fresh replacement Pods retrieved retained L2 data at TP=1 and TP=2; this is supplemental evidence. | +| Role test | `kv_consumer` still stored and `kv_producer` still retrieved. LMCache 0.5.3 does not enforce the rendered role. | + +The runtime-owner wheel overlay is validation scaffolding, not authority for +CacheBackend to modify engine images. The fixture also lacked an HTTP readiness +probe, so traffic tests waited for the engine health endpoint. + +- [x] Host-only TP=1/2 GPU store → local-cache reset → MP L1 retrieve. +- [x] Live KV events, index/status, shared-memory budget, and native-extension + checks pass after their discovered defects were repaired. +- [x] ReadWrite performs both store and retrieve. +- [x] ReadOnly/WriteOnly negative testing proves LMCache 0.5.3 ignores + directionality even though the adapter renders the requested role. +- [x] Supplemental cross-Pod Redis retrieval passes at TP=1/2. +- [x] Admission rejects unsupported parallel shapes and injects typed vLLM MP + without capability annotations. +- [x] LMCache 0.5.3 exposes no supported public heartbeat/MQ tuning surface; + the API intentionally relies on its internal defaults. +- [x] Reject ReadOnly/WriteOnly for every LMCache backend because the validated + connector does not enforce them; defer directional roles to an independent + future connector capability. ### Exit criteria -- vLLM MP passes the required GPU matrix below. -- The replacement provides a documented migration path for host-only IP and - centralized `lm://` users. -- MP becomes the recommended path in samples and operator docs. Legacy IP stays - implementation-only until repository migration and is then removed directly; - Phase 6 applies only if a legacy consumer appears before removal. +- [x] Required TP=1 and TP=2 host-only GPU paths pass. +- [x] Role semantics are safe and accurately represented by the API: LMCache + admits only the validated ReadWrite behavior. ## Phase 5 — migration tooling and consumer migration -### Goal +- **Status:** Not started +- **Depends on:** Phase 4 -Make every legacy object's semantic change explicit. No automated migration may -silently remove cross-Pod cache sharing or select a different remote L3. +### Objective -### Conditional tooling +Convert repository-owned consumers to MP without silently changing cross-Pod +sharing or remote-L3 semantics. -Phase 0 found no external consumers or installed legacy objects, so migration -tooling is not a default deliverable. Build the following only if that fact -changes before removal: +### Scope + +Repository migration is required. Inventory/migration tooling is conditional +because Phase 0 found no external users or installed legacy objects. + +### Deliverables -- [ ] Add a read-only inventory/doctor command that classifies every legacy - `CacheBackend` and prints its migration class. -- [ ] Add a dry-run manifest migration command or documented deterministic - transformation. -- [ ] Report fields that cannot be mapped automatically. -- [ ] Emit `LegacyInProcessDeprecated` status/Events for remaining legacy - objects. -- [ ] Provide rollback instructions during the compatibility window. +- [ ] Convert canonical samples, reference-stack manifests, support tables, CLI + output, documentation, screenshots, and non-transition fixtures to MP. +- [ ] Remove language that presents the legacy LMCache server as a CPU profile + or default backend. +- [ ] Reconfirm the zero-external-consumer assumption before removal. +- [ ] If external consumers appear, add inventory/doctor, dry-run conversion, + unmappable-field reporting, deprecation Events, and rollback guidance. -### Migration classes +Migration rules: | Existing object | Automatic portion | Required operator choice | |---|---|---| @@ -905,72 +708,72 @@ changes before removal: | vLLM IP + existing engine-side Mooncake provider | Preserve local intent only. | Wait for MP + Mooncake Store L3 support or migrate explicitly to Redis; URL config is not equivalent to MP adapter config. | | Any IP object with `remoteSerde` | None. | Remove it or map it to a future typed L3 serde only when that adapter supports and validates the same semantics. | -### Repository migration +### Validation -- [ ] Convert every canonical sample to MP. -- [ ] Convert reference-stack manifests. -- [ ] Replace IP documentation and screenshots. -- [ ] Replace IP unit/integration fixtures where they are not explicitly testing - the transition or a conditional Phase 6 compatibility window. -- [ ] Update support tables and CLI output. -- [ ] Remove language that calls the legacy LMCache server a CPU profile or the - default LMCache backend. +- [ ] Repository search finds no repository-owned production LMCache workload + still using IP, `lm://`, `LMCacheServer`, or flat SGLang MP fields. +- [ ] Migrated samples and reference manifests pass admission/default-install + smoke. +- [ ] Any newly discovered legacy object has an owner and explicit disposition. ### Exit criteria -- Every repository-owned LMCache workload uses MP. -- If any external legacy object appears, it has an owner and migration - disposition. -- If migration tooling becomes necessary, it reports zero unknown/unclassified - legacy shapes. -- No migration silently changes cross-Pod sharing behavior. +- [ ] Every repository-owned LMCache workload uses MP. +- [ ] No migration silently changes cross-Pod sharing behavior. +- [ ] Conditional tooling, if activated, reports zero unknown legacy shapes. ## Phase 6 — reject new IP objects -**Conditional:** Phase 0 found no external consumers or installed legacy -objects. Skip this phase and proceed from Phase 5 to Phase 7 if that remains true. -Activate it in full if a legacy consumer or object appears before removal. +- **Status:** Conditional; currently not required +- **Depends on:** Phase 5 + +### Objective -### Goal +If a legacy consumer appears, stop the legacy population from growing while it +is migrated or deleted. -Stop growth of the legacy population while allowing controlled migration or -deletion of existing objects. +### Scope -### Admission policy +Skip directly to Phase 7 if the Phase 0 zero-consumer finding still holds. +Otherwise this phase adds a temporary compatibility gate, not new IP features. -- [ ] Reject creation of vLLM LMCache objects without the MP block. -- [ ] Reject creation of `remoteStorage.provider: LMCacheServer`. -- [ ] Reject reintroduction of removed legacy fields. -- [ ] Grandfather existing IP objects only for: - - read/status; - - deletion; - - updates required to migrate to MP. -- [ ] Reject updates that scale out, materially retune, or otherwise extend the - lifetime/scope of a legacy IP deployment. +### Deliverables -### Operational gates +- [ ] Reject new vLLM LMCache objects without typed MP. +- [ ] Reject new `LMCacheServer` providers and reintroduced legacy fields. +- [ ] Permit existing IP objects only for read/status, deletion, and migration; + reject scale-out or lifetime-extending updates. +- [ ] Report remaining legacy count and emit migration-linked warnings. +- [ ] Publish the physical-removal target and observation window. + +### Validation -- [ ] CLI/doctor reports remaining legacy object count. -- [ ] Release notes state the physical-removal target release. -- [ ] Warning Events link to migration documentation. -- [ ] A defined observation window passes with zero newly created IP objects. +- [ ] CREATE/UPDATE admission tests cover rejection and grandfather rules. +- [ ] Observation window records zero newly created IP objects. +- [ ] Every exception has an owner and expiry. ### Exit criteria -- No supported API path can create a new IP data plane. -- Remaining legacy objects are zero, or each has an approved time-bounded - exception. -- MP error rate, hit behavior, and recovery behavior meet the agreed production - baseline. +- [ ] No supported API path can create a new IP data plane. +- [ ] Legacy count is zero or every exception is time-bounded. +- [ ] MP production health meets the agreed baseline. ## Phase 7 — remove IP and the legacy LMCache server -### Goal +- **Status:** Not started +- **Depends on:** Phase 5; Phase 6 only if activated -Delete the project-deprecated data plane and all code that exists solely to -operate it. +### Objective -### Code removal +Delete the IP data plane and all code/schema that exists only to support it. + +### Scope + +Includes runtime adapters, provider protocols, controller workloads, status, +samples, tests, and legacy API fields. Historical migration documentation may +remain when clearly marked. + +### Deliverables - [ ] Remove the vLLM legacy LMCache adapter. - [ ] Remove `LMCacheConnectorV1` rendering. @@ -979,21 +782,16 @@ operate it. - [ ] Remove `ProtocolLMCache` and the `lm://` endpoint parser/binding. - [ ] Remove the managed and external `LMCacheServer` provider surface. - [ ] Remove the standalone LMCache-server workload renderer. -- [ ] Remove the server-instance restart cascade if no remaining provider needs - it; otherwise narrow and rename it to the actual capability. - [ ] Remove IP-only status fields, metrics, Events, samples, and tests. - [ ] Remove compatibility defaulting/validation and migration-only code after - the supported migration window closes. - -### API cleanup - + any supported migration window closes. - [ ] Remove legacy flat LMCache fields after their replacement is complete. - [ ] Remove `LMCacheServer` from CRD enums and provider-specific schema. - [ ] Remove or relocate top-level managed-provider workload fields according to the Phase 0 decision. - [ ] Regenerate CRDs, deepcopy code, examples, and reference documentation. -### Verification +### Validation - [ ] `go test ./...` passes. - [ ] `make verify-samples` passes. @@ -1004,24 +802,30 @@ operate it. - `ProtocolLMCache`; - `lm://`; - the managed `LMCacheServer` provider. -- [ ] Migration documentation may retain historical references clearly marked as - removed. +- [ ] Any retained historical reference is clearly marked as removed behavior. ### Exit criteria -- Only MP adapters can be selected for `spec.type: LMCache`. -- No controller workload or engine wire implements IP. -- No new or stored object requires the legacy schema to reconcile. +- [ ] Only MP adapters can be selected for `spec.type: LMCache`. +- [ ] No controller workload or engine wire implements IP. +- [ ] No supported stored object requires the legacy schema. ## Phase 8 — NodeLocal shared MP servers -### Goal +- **Status:** Not started +- **Depends on:** Phases 3–4; does not block Phase 7 + +### Objective + +Allow multiple engine Pods of one `CacheBackend` to share one MP server per +node without weakening placement, isolation, or status correctness. -Add the higher-efficiency topology in which multiple engine Pods of one -`CacheBackend` share a node-local MP server without weakening placement, -isolation, or status correctness. +### Scope -### Controller work +Includes same-node discovery, DaemonSet lifecycle, shared capacity, and +multi-node coverage. Cross-`CacheBackend` sharing remains out of scope. + +### Deliverables - [ ] Reconcile one DaemonSet per NodeLocal `CacheBackend`. - [ ] Restrict it to intended GPU/engine nodes through typed scheduling fields. @@ -1032,18 +836,12 @@ isolation, or status correctness. - [ ] Compute desired/ready servers and engine-node coverage. - [ ] Handle engine scheduling before the node-local server is ready without starting an engine against a missing required MP endpoint. - -### Engine injection - - [ ] Derive the node-local address from the engine Pod's node/host IP through a Downward API field or another deterministic node-scoped mechanism. - [ ] Do not use a load-balanced ClusterIP as the CUDA MP endpoint. - [ ] Keep SGLang and vLLM launch surfaces engine-specific. - [ ] Validate the server's global chunk size and version against every selected engine Pod. - -### Isolation and resource work - - [ ] Define port-conflict behavior for multiple NodeLocal CacheBackends on one node. - [ ] Restrict the first implementation to one trust/tenant domain per @@ -1051,14 +849,14 @@ isolation, or status correctness. - [ ] Document that L1 capacity is per node and shared by selected engine Pods. - [ ] Size `maxGPUWorkers` for the number of engine instances sharing a server. - [ ] Add NetworkPolicy/firewall guidance where host networking permits it. -- [ ] Assess the security impact of exposing all node GPUs to the MP server. +- [ ] Assess the security impact of host networking/shared memory and GPU + visibility. ### Validation - [ ] One engine Pod on one node. - [ ] Multiple engine Pods sharing one node-local server. - [ ] Engines spread across multiple nodes, each using only its local server. -- [ ] DaemonSet rollout and single-node server restart. - [ ] Node drain and engine rescheduling. - [ ] Host-port conflict negative test. - [ ] Redis outage/recovery with multiple node-local servers. @@ -1066,13 +864,108 @@ isolation, or status correctness. ### Exit criteria -- Every selected engine Pod is covered by exactly one healthy local MP server. -- No generic Service load balancing can route an engine to another node's MP - server. -- Shared L1 behavior, resource accounting, and failure blast radius are measured - and documented. -- Cross-`CacheBackend` pool sharing remains rejected until a separate resource - and tenancy model is approved. +- [ ] Every selected engine Pod is covered by exactly one healthy same-node MP + server. +- [ ] No load-balanced Service can route an engine to another node's server. +- [ ] Shared L1 accounting and failure blast radius are measured. +- [ ] Cross-`CacheBackend` sharing remains rejected. + +## Post-migration improvements and additional features + +These items are separate capability profiles. They are not Phase 3 or Phase 4 +exit criteria and do not block migration away from the legacy IP data plane: + +- [ ] Design and validate multi-node TP and vLLM distributed-executor profiles, + including connector/server cardinality, endpoint discovery, failure + domains, scheduling, and an explicit admission contract. +- [ ] Design and validate MLA and other model-specific connector profiles using + model architecture metadata rather than image or model-name heuristics. +- [ ] Add client/server compatibility signaling or health detection before + supporting multiple LMCache version baselines; do not generalize from an + arbitrary mismatched-version test pair. +- [ ] Add each profile to the supported validation matrix only after its own + GPU correctness, failure-recovery, and operability gates pass. + +### Directional LMCache roles for PD separation + +`ReadOnly` / `WriteOnly` remain generic CacheBackend API concepts, but all +LMCache backends currently admit only `ReadWrite`. This is an intentional safety +restriction, not a claim that producer/consumer roles are unnecessary. + +| Finding | Evidence/impact | +|---|---| +| SGLang's LMCache integration has no directional role surface. | `--enable-lmcache` always participates in both store and retrieve. | +| vLLM accepts `kv_consumer`, `kv_producer`, and `kv_both`. | These are connector configuration values, not LMCache server roles. | +| LMCache 0.5.3's vLLM MP connector did not enforce the configured direction in live GPU tests. | `kv_consumer` still stored and `kv_producer` still retrieved, so exposing ReadOnly/WriteOnly would create a false API guarantee. | + +Future work must treat directional access as a separately validated connector +capability: + +- [ ] Define PD producer, consumer, and optional decode write-back semantics, + including whether generated-token KV may be persisted after a request. +- [ ] Adopt a pinned connector that prevents store in consumer mode and retrieve + in producer mode rather than relying only on configuration naming. +- [ ] Add GPU negative tests that fail on any prohibited request, plus normal + prefill-to-decode transfer and multi-turn write-back tests where selected. +- [ ] Lift LMCache admission restrictions only for an adapter/version profile + that passes those tests; do not infer support from engine CLI acceptance. + +### LMCache connector control-plane convergence and SGLang TP>1 + +SGLang TP>1 is outside the migration baseline. Inference-cache neither patches +the engine-owned connector nor adds a TP=1 admission guard. + +| Finding | Evidence/impact | +|---|---| +| vLLM has one scheduler-side owner for LOOKUP/status/session state. | Per-rank workers only retrieve their GPU shard. | +| SGLang 0.5.3 runs the control flow in every TP rank. | In TP=2, one rank consumed the exactly-once prefetch result; the other got `Prefetch job ... not found`, so the cross-rank minimum became zero. | +| A diagnostic rank-0-owner overlay retrieved 1,280 tokens on both ranks. | It proves a coordination direction, not a safe production patch; collective failure/cancellation remains undesigned. | + +Future work must answer the architectural question before selecting a fix: + +- [ ] Determine why the vLLM and SGLang integrations deliberately use different + scheduler/worker ownership models and whether SGLang exposes a stable + scheduler-to-worker metadata path suitable for LMCache. +- [ ] Define one owner for LOOKUP, prefetch status, sessions, and global lock + cleanup while preserving per-rank registration and GPU RETRIEVE. +- [ ] Define bounded cross-rank error and cancellation propagation so a failed + owner cannot hang peers in a collective. +- [ ] Add upstream TP=2 tests covering miss/store, GPU flush, host-only hit, + Redis-backed hit, partial hit, timeout, cancellation, and lock/session + cleanup. +- [ ] Adopt only an immutable released connector artifact, then add SGLang + TP>1 back to the production validation matrix after the tests pass. + +### LMCache MP server restart and connector re-registration + +The migration guarantees steady-state MP operation and sidecar process-health +observation. It does not guarantee that a running engine continues caching after +its MP server restarts, and it does not patch LMCache 0.5.3 to add that behavior. + +| Finding | Evidence/impact | +|---|---| +| Kubernetes can restart the native sidecar independently. | `ConnectorReady` follows process health, but process recovery does not prove registration recovery. | +| SGLang TP=1 did not recover registration. | The new server had no GPU context and the next request hung; LMCache 0.5.3's SGLang adapter does not register the vLLM adapter's recovery callback. | +| vLLM recovered after a 70-second outage. | Re-registration and post-recovery store/retrieve passed. | +| vLLM failed after a fast 10–15-second restart. | Heartbeat missed the outage, so the new server lost registration without triggering recovery. | + +If this capability is selected later, its independent scope is: + +- [ ] Decide whether the supported policy is sidecar-only recovery or complete + engine-Pod recreation; document the availability and latency trade-off. +- [ ] If sidecar-only recovery is selected, implement or adopt a pinned LMCache + client/server version that re-registers SGLang and vLLM GPU contexts and + detects server replacement even when no heartbeat lands in the outage + window. +- [ ] Invalidate pre-restart lookup/session state and prove bounded recompute + rather than a hung or partially restored request. +- [ ] Make connector status registration-aware so an empty recovered server is + not reported as `ConnectorReady=True`. +- [ ] Validate crash, hang, fast restart, repeated restart, callback failure, + TP=1/TP=2, and post-recovery store/flush/retrieve for each selected engine + profile. +- [ ] For NodeLocal, validate DaemonSet rollout and single-node server restart + separately from the basic same-node topology. ## Required GPU validation matrix @@ -1083,12 +976,8 @@ not sufficient. | Runtime | Topology | Remote L3 | Parallelism | Required by | |---|---|---|---|---| | SGLang | PodLocal | none | TP=1 | Phase 3 | -| SGLang | PodLocal | Redis | TP=1 | Phase 3 | -| SGLang | PodLocal | Redis | TP=2 | Phase 3 | | vLLM | PodLocal | none | TP=1 | Phase 4 | -| vLLM | PodLocal | Redis | TP=1 | Phase 4 | -| vLLM | PodLocal | Redis | TP=2 | Phase 4 | -| vLLM | PodLocal | Redis | TP=4 | Before production recommendation for common multi-GPU workloads | +| vLLM | PodLocal | none | TP=2 | Phase 4 | | SGLang | NodeLocal | Redis | multiple engine Pods | Phase 8 | | vLLM | NodeLocal | Redis | multiple engine Pods | Phase 8 | @@ -1100,8 +989,8 @@ Every required data test records: - first-request store evidence; - GPU-cache clear or fresh-engine proof; - second-request retrieve/hit evidence; -- MP and L3 metrics before and after; -- failure/recovery timestamps where applicable. +- MP metrics before and after, plus L3 metrics only when an optional L3 binding + is part of that particular test. ## Test pyramid @@ -1112,7 +1001,7 @@ Every required data test records: | Envtest | real CREATE/UPDATE admission and status persistence; legacy grandfathering only if Phase 6 activates | | Kubernetes smoke | live webhook injection into matching engine Pods, native-sidecar schema support, controller-owned workload shape | | GPU functional | store/flush/retrieve and cross-Pod L3 reuse | -| GPU fault | MP crash/hang/restart, Redis loss/recovery, engine rollout, node drain for NodeLocal | +| GPU fault | Redis loss/recovery, engine rollout, node drain for NodeLocal | | Upgrade/migration | Repository manifest conversion by default; old-object inventory, dry-run conversion, grandfather rules, and rollback only if Phase 6 activates | ## Security, reliability, scalability, and cost gates @@ -1121,7 +1010,7 @@ Every required data test records: - [ ] No production managed Redis profile is exposed without an explicit network isolation and credential/TLS posture. -- [ ] Secrets are referenced, not embedded in CR status, Pod args visible to all +- [x] Secrets are referenced, not embedded in CR status, Pod args visible to all readers, logs, or Events. - [ ] PodLocal and NodeLocal GPU visibility is documented and reviewed for the target tenancy model. @@ -1130,26 +1019,23 @@ Every required data test records: ### Reliability -- [ ] MP server health affects connector status. -- [ ] A hung process is detected, not just an exited process. -- [ ] Recovery cannot leave the engine Ready with permanently disabled caching. +- [x] MP server health affects connector status. - [ ] Remote L3 loss follows tested fail-open/fail-closed behavior. -- [ ] Restart actions are scoped to the failing component's capability. ### Scalability and latency -- [ ] PodLocal memory cost is reported per engine Pod. +- [x] PodLocal memory cost is reported per engine Pod. - [ ] NodeLocal memory cost is reported per node. - [ ] Worker pool sizing is tested under the expected engine count and TP shape. - [ ] Remote L3 concurrency and connection limits are bounded. -- [ ] Routing/index signals can be correlated with actual LMCache hit metrics. +- [x] Routing/index signals can be correlated with actual LMCache hit metrics. ### Operability -- [ ] Status distinguishes connector, MP server, engine, and remote L3 health. +- [x] Status distinguishes connector, MP server, engine, and remote L3 health. - [ ] Metrics expose server availability, L1/L3 store/retrieve/hit, capacity, - eviction, and recovery. -- [ ] Events contain an actionable recovery or migration instruction. + and eviction. +- [ ] Events contain an actionable remediation or migration instruction. - [ ] Samples never depend on an implicit runtime-selected connector mode. ## Risk register @@ -1157,9 +1043,8 @@ Every required data test records: | Risk | Impact | Mitigation / gate | |---|---|---| | A legacy consumer appears after Phase 0 | Breaking removal | Reconfirm before removal; activate Phase 6 and a grandfather period when non-zero. | -| MP client/server version skew | Permanent unhealthy or protocol failure | Same-image default for PodLocal, pinned matrix, skew-negative tests. | -| Worker restart does not re-register engine state | Ready engine silently misses forever | Runtime-specific recovery test and condition before Phase 3/4 exit. | -| Redis restart rolls all engines | Availability blast radius | Capability-specific restart policy in Phase 2. | +| MP client/server version skew | Permanent unhealthy or protocol failure | Pin a validated client/server baseline and record exact artifacts; automatic version negotiation/detection is a future improvement. | +| Redis restart rolls all engines | Availability blast radius | Lifecycle-specific restart policy in Phase 2. | | `failOpen` is only a custom env | Contract not enforced | Render native runtime policy and fault-test it. | | Sidecar sees all node GPUs | Isolation exposure | Document/review tenant model; prefer dedicated nodes where required. | | PodLocal duplicates CPU L2 | Memory cost per replica | Explicit per-Pod capacity; NodeLocal follow-up. | @@ -1168,39 +1053,31 @@ Every required data test records: | Existing engine-side Mooncake config is treated as MP-equivalent | Admission succeeds but adapter cannot start | No automatic migration; separate MP + Mooncake Store implementation. | | Index says warm while MP/L3 evicted data | Routing quality degrades silently | Correlate cache events with LMCache metrics/health; define stale-entry behavior. | -## Phase tracking template - -Each implementation PR updates the delivery table and its phase checklist in the -same change. A phase is not marked complete merely because code merged. +## Roadmap maintenance -```markdown -### Phase N status update — YYYY-MM-DD - -- Status: not started | in progress | blocked | complete -- Owner: -- Tracking issue: -- PRs: -- Validation artifacts: -- Remaining exit criteria: -- Decision changes: -``` +Each implementation PR updates the delivery table and the affected phase's +checkboxes. A phase becomes complete only when every exit criterion is checked; +validation details stay summarized in the phase evidence table rather than in +dated closure sections or separate phase documents. ## Overall definition of done The migration is complete only when all of the following are true: - [ ] `spec.type: LMCache` selects only MP implementations. -- [ ] Both SGLang and vLLM pass the required PodLocal GPU matrix. -- [ ] Host-only and Redis-backed MP are supported for both engines. -- [ ] MP server health, failure, and recovery are observable and tested. +- [x] Both SGLang and vLLM pass the required PodLocal GPU matrix. +- [x] Host-only MP is supported for both engines; optional L3 implementations + are validated and versioned independently from the engine connector gate. +- [x] Current MP server health is observable and steady-state cache behavior is + tested. - [ ] `remoteStorage` is optional L3 and no longer contains LMCacheServer. - [ ] No production code injects `LMCacheConnectorV1`, `lm://`, or `LMCACHE_REMOTE_URL`. -- [ ] No generic managed-provider restart automatically rolls MP engines. +- [x] Remote-L3 lifecycle events do not automatically roll MP engines. - [ ] Every old IP object has been migrated or intentionally deleted. - [ ] Canonical samples, reference manifests, CLI output, and design documents describe only the implemented MP behavior. -- [ ] NodeLocal, if enabled, guarantees same-node server selection and accurate +- [x] NodeLocal, if enabled, guarantees same-node server selection and accurate engine coverage; otherwise it remains rejected rather than partially accepted. diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index a3e8ef7b..e3c4169a 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -2279,9 +2279,6 @@ metadata: name: sglang-podlocal-admission labels: inferencecache.io/runtime: sglang-mp - annotations: - inferencecache.io/lmcache-connector-profile: sglang-lmcache-mp-v1 - inferencecache.io/lmcache-client-version: "0.5.3" spec: nodeSelector: inferencecache.io/install-smoke-never-schedule: "true" @@ -2320,7 +2317,7 @@ typed_engine_experimental="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICA typed_engine_mounts="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ -o jsonpath='{.spec.containers[?(@.name=="sglang")].volumeMounts[*].mountPath}' 2>/dev/null || true)" -expected_mp_image="lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13" +expected_mp_image="docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13" if [ "$typed_injected_by" != "$CANONICAL_SMOKE_NS/$CANONICAL_TYPED_CB" ] || \ [ "$typed_metrics_label" != "true" ] || \ [ "$typed_server_image" != "$expected_mp_image" ] || \ @@ -2373,9 +2370,6 @@ metadata: name: vllm-podlocal-admission labels: inferencecache.io/runtime: vllm-mp - annotations: - inferencecache.io/lmcache-connector-profile: vllm-lmcache-mp-v1 - inferencecache.io/lmcache-client-version: "0.5.3" spec: nodeSelector: inferencecache.io/install-smoke-never-schedule: "true" diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go index 27a3a60f..61a04db6 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go @@ -31,6 +31,7 @@ const ( lmCacheMPShmVolumeName = "lmcache-mp-shm" lmCacheMPShmMountPath = "/dev/shm" + lmCacheMPShmHeadroom = "1Gi" lmCacheMPServerPortName = "lmcache-mp" lmCacheMPHTTPPortName = "lmcache-http" @@ -112,24 +113,27 @@ func renderLMCachePodLocalServer(pod *corev1.PodSpec, engineContainerName string }) } + shmBudget := lmCacheMPMemoryBudget(cfg.L1Capacity) shmMount := corev1.VolumeMount{Name: lmCacheMPShmVolumeName, MountPath: lmCacheMPShmMountPath} if existing := mountAtPath(engine.VolumeMounts, lmCacheMPShmMountPath); existing != nil && !(owned && existing.Name == lmCacheMPShmVolumeName) { if err := checkLMCacheMPShmReusable(work.Volumes, *existing); err != nil { return "", err } + if err := checkLMCacheMPShmBudget(work.Volumes, *existing, shmBudget); err != nil { + return "", err + } shmMount = corev1.VolumeMount{ Name: existing.Name, MountPath: lmCacheMPShmMountPath, SubPath: existing.SubPath, } } else { - l1 := cfg.L1Capacity.DeepCopy() work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ Name: lmCacheMPShmVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ Medium: corev1.StorageMediumMemory, - SizeLimit: &l1, + SizeLimit: &shmBudget, }}, }, owned) if err != nil { @@ -154,6 +158,38 @@ func renderLMCachePodLocalServer(pod *corev1.PodSpec, engineContainerName string return "", nil } +func lmCacheMPMemoryBudget(l1Capacity resource.Quantity) resource.Quantity { + budget := l1Capacity.DeepCopy() + budget.Add(resource.MustParse(lmCacheMPShmHeadroom)) + return budget +} + +// checkLMCacheMPShmBudget validates an engine-owned /dev/shm volume without +// mutating it. A larger operator-owned tmpfs is safe to share; an unbounded, +// disk-backed, or undersized volume cannot satisfy the typed L1 contract. +func checkLMCacheMPShmBudget(volumes []corev1.Volume, mount corev1.VolumeMount, budget resource.Quantity) error { + for i := range volumes { + if volumes[i].Name != mount.Name { + continue + } + emptyDir := volumes[i].EmptyDir + if emptyDir == nil || emptyDir.Medium != corev1.StorageMediumMemory { + return fmt.Errorf("render LMCache MP server: engine container mounts %q from volume %q, but PodLocal L1 requires a memory-backed emptyDir", lmCacheMPShmMountPath, mount.Name) + } + if emptyDir.SizeLimit == nil || emptyDir.SizeLimit.Cmp(budget) < 0 { + var got string + if emptyDir.SizeLimit == nil { + got = "unbounded" + } else { + got = emptyDir.SizeLimit.String() + } + return fmt.Errorf("render LMCache MP server: engine /dev/shm volume %q has sizeLimit %s, need at least %s (l1Capacity + %s headroom)", mount.Name, got, budget.String(), lmCacheMPShmHeadroom) + } + return nil + } + return fmt.Errorf("render LMCache MP server: engine /dev/shm mount references missing volume %q", mount.Name) +} + func validateLMCacheMPServerConfig(cfg lmCacheMPServerConfig) error { switch { case strings.TrimSpace(cfg.Image) == "": diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go index 6ace3ff0..f4d99a19 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go @@ -109,8 +109,8 @@ func TestRenderLMCachePodLocalServerGolden(t *testing.T) { t.Fatalf("server ports = %+v", server.Ports) } shm := findVolume(pod.Volumes, lmCacheMPShmVolumeName) - if shm == nil || shm.EmptyDir == nil || shm.EmptyDir.Medium != corev1.StorageMediumMemory || shm.EmptyDir.SizeLimit == nil || shm.EmptyDir.SizeLimit.Cmp(resource.MustParse("4Gi")) != 0 { - t.Fatalf("shared-memory volume = %+v, want bounded 4Gi tmpfs", shm) + if shm == nil || shm.EmptyDir == nil || shm.EmptyDir.Medium != corev1.StorageMediumMemory || shm.EmptyDir.SizeLimit == nil || shm.EmptyDir.SizeLimit.Cmp(resource.MustParse("5Gi")) != 0 { + t.Fatalf("shared-memory volume = %+v, want bounded 5Gi tmpfs (4Gi L1 + 1Gi headroom)", shm) } if findVolume(pod.Volumes, lmCacheMPConfigVolumeName) == nil { t.Fatalf("client config volume missing: %+v", pod.Volumes) @@ -216,7 +216,10 @@ func TestRenderLMCachePodLocalServerReusesWritableShm(t *testing.T) { Name: "engine-shm", MountPath: "/dev/shm", SubPath: "shared", }}, }}, - Volumes: []corev1.Volume{{Name: "engine-shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}}}, + Volumes: []corev1.Volume{{Name: "engine-shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: func() *resource.Quantity { q := resource.MustParse("6Gi"); return &q }(), + }}}}, } if _, err := renderLMCachePodLocalServer(pod, "engine", testLMCacheMPConfig()); err != nil { t.Fatalf("renderLMCachePodLocalServer: %v", err) @@ -230,6 +233,51 @@ func TestRenderLMCachePodLocalServerReusesWritableShm(t *testing.T) { } } +func TestRenderLMCachePodLocalServerRejectsUnsafeExistingShmBudget(t *testing.T) { + tests := []struct { + name string + source corev1.VolumeSource + want string + }{ + { + name: "unbounded memory emptyDir", + source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}, + want: "sizeLimit unbounded", + }, + { + name: "undersized memory emptyDir", + source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: func() *resource.Quantity { q := resource.MustParse("4Gi"); return &q }(), + }}, + want: "need at least 5Gi", + }, + { + name: "disk-backed emptyDir", + source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: func() *resource.Quantity { q := resource.MustParse("6Gi"); return &q }(), + }}, + want: "memory-backed emptyDir", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pod := &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine", VolumeMounts: []corev1.VolumeMount{{Name: "engine-shm", MountPath: "/dev/shm"}}}}, + Volumes: []corev1.Volume{{Name: "engine-shm", VolumeSource: tc.source}}, + } + before := pod.DeepCopy() + _, err := renderLMCachePodLocalServer(pod, "engine", testLMCacheMPConfig()) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("error = %v, want %q", err, tc.want) + } + if !reflect.DeepEqual(pod, before) { + t.Fatalf("failed render mutated pod\nbefore=%+v\nafter=%+v", before, pod) + } + }) + } +} + func TestRenderLMCacheMPL2BindingRejectsUnsupportedV053Features(t *testing.T) { db := int32(1) for _, tc := range []struct { diff --git a/internal/adapters/builtin/runtime/lmcachecheck.go b/internal/adapters/builtin/runtime/lmcachecheck.go index 66366893..24c39dfb 100644 --- a/internal/adapters/builtin/runtime/lmcachecheck.go +++ b/internal/adapters/builtin/runtime/lmcachecheck.go @@ -16,17 +16,20 @@ import ( // it wants a GPU. Auto mode skips CPU-only engines. const gpuResourceName = corev1.ResourceName("nvidia.com/gpu") -// kernelCheckScript is the Python the init container runs against the engine -// image. It locates the package dir WITHOUT executing lmcache.__init__ (which -// swallows the c_ops failure into a WARNING and overrides +// kernelCheckScript is the Python the init container runs against the vLLM +// engine image. It first locates LMCache WITHOUT executing lmcache.__init__ +// (which swallows the c_ops failure into a WARNING and overrides // sys.modules["lmcache.c_ops"] with a fallback shim, so a naive // `import lmcache.c_ops` ALWAYS succeeds — a silent no-op). Instead it // dlopens the native c_ops*.so from disk via ctypes.CDLL, which re-does the -// real dynamic load and raises on a missing/mismatched libcudart (empirically: -// "OSError: libcudart.so.13: cannot open shared object file"). torch MUST be -// imported first — the extension DT_NEEDs libtorch's libc10.so. +// real dynamic load and raises on a missing/mismatched dependency. It then +// imports the vLLM core extension used by the installed vLLM generation: +// vllm._C_stable_libtorch in current stable-ABI builds, with vllm._C as the +// legacy fallback. This covers the engine's own CUDA dependency boundary too +// (empirically, that extension alone can require libcudart.so.13). torch MUST +// be imported first — the extensions DT_NEED libtorch's libc10.so. const kernelCheckScript = ` -import sys, os, glob, importlib.util, ctypes +import sys, os, glob, importlib, importlib.util, ctypes STRICT = os.environ.get("KERNEL_CHECK_STRICT") == "1" MSG = "/dev/termination-log" def emit(s): @@ -56,6 +59,18 @@ try: # CDLL needs no init symbol — it tests exactly the dlopen/DT_NEEDED # resolution where the kernel/CUDA mismatch lives. ctypes.CDLL(sos[0]) + # LMCache c_ops loading alone does not certify the surrounding vLLM image: + # vLLM's core extension can carry a different libcudart dependency. Stable + # ABI wheels use _C_stable_libtorch; older wheels use _C. Probe in that + # order and import the first extension actually shipped by the image. + core = None + for candidate in ("vllm._C_stable_libtorch", "vllm._C"): + if importlib.util.find_spec(candidate) is not None: + core = candidate + break + if core is None: + fail("no supported vLLM native core extension present (tried vllm._C_stable_libtorch, vllm._C)") + importlib.import_module(core) emit("OK") except SystemExit: raise diff --git a/internal/adapters/builtin/runtime/lmcachecheck_script_test.go b/internal/adapters/builtin/runtime/lmcachecheck_script_test.go index 4ba754e9..f8b85f5b 100644 --- a/internal/adapters/builtin/runtime/lmcachecheck_script_test.go +++ b/internal/adapters/builtin/runtime/lmcachecheck_script_test.go @@ -114,9 +114,10 @@ func TestKernelCheckScriptLmcacheAbsentReportsFail(t *testing.T) { // makeHealthyLmcachePkg builds a synthetic PYTHONPATH root with a loadable // native c_ops*.so (compiled trivially — no Python init symbol, no CUDA deps) -// plus a stub `torch` package so the detector's `import torch` succeeds. This -// reproduces a HEALTHY engine for the detector: lmcache present, c_ops present, -// dlopen-able. It is the regression guard for the OK path — with the previous +// plus stub `torch` and current `vllm._C_stable_libtorch` packages so both +// native compatibility checks succeed. This reproduces a HEALTHY engine for +// the detector: LMCache present, c_ops dlopen-able, and the vLLM extension +// importable. It is the regression guard for the OK path — with the previous // importlib.exec_module loader (which derives PyInit_), loading this // header-free .so would have FAILED; ctypes.CDLL loads it, so this asserts the // detector reports OK on a kernel that actually loads. Skips if no C compiler. @@ -137,10 +138,18 @@ func makeHealthyLmcachePkg(t *testing.T) string { if err := os.MkdirAll(filepath.Join(root, "torch"), 0o755); err != nil { t.Fatal(err) } + if err := os.MkdirAll(filepath.Join(root, "vllm"), 0o755); err != nil { + t.Fatal(err) + } if err := os.MkdirAll(pkg, 0o755); err != nil { t.Fatal(err) } - for _, f := range []string{filepath.Join(pkg, "__init__.py"), filepath.Join(root, "torch", "__init__.py")} { + for _, f := range []string{ + filepath.Join(pkg, "__init__.py"), + filepath.Join(root, "torch", "__init__.py"), + filepath.Join(root, "vllm", "__init__.py"), + filepath.Join(root, "vllm", "_C_stable_libtorch.py"), + } { if err := os.WriteFile(f, []byte("# stub\n"), 0o644); err != nil { t.Fatal(err) } @@ -157,6 +166,37 @@ func makeHealthyLmcachePkg(t *testing.T) string { return root } +func TestKernelCheckScriptMissingVLLMNativeExtensionReportsFail(t *testing.T) { + root := makeHealthyLmcachePkg(t) + if err := os.Remove(filepath.Join(root, "vllm", "_C_stable_libtorch.py")); err != nil { + t.Fatal(err) + } + msg, code := runScript(t, root, false) + if code != 0 { + t.Errorf("report-only exit = %d, want 0", code) + } + if !strings.Contains(msg, "no supported vLLM native core extension") { + t.Errorf("message = %q, want missing native-core failure", msg) + } +} + +func TestKernelCheckScriptLegacyVLLMNativeExtensionFallbackReportsOK(t *testing.T) { + root := makeHealthyLmcachePkg(t) + if err := os.Remove(filepath.Join(root, "vllm", "_C_stable_libtorch.py")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "vllm", "_C.py"), []byte("# legacy stub\n"), 0o644); err != nil { + t.Fatal(err) + } + msg, code := runScript(t, root, false) + if code != 0 { + t.Errorf("exit = %d, want 0", code) + } + if strings.TrimSpace(msg) != enginebinding.KernelCheckMsgOK { + t.Errorf("message = %q, want %q for legacy fallback", msg, enginebinding.KernelCheckMsgOK) + } +} + func TestKernelCheckScriptHealthyExtensionReportsOK(t *testing.T) { msg, code := runScript(t, makeHealthyLmcachePkg(t), false) if code != 0 { diff --git a/internal/adapters/builtin/runtime/sglang_hicache.go b/internal/adapters/builtin/runtime/sglang_hicache.go index 060ef3dd..9fb64d50 100644 --- a/internal/adapters/builtin/runtime/sglang_hicache.go +++ b/internal/adapters/builtin/runtime/sglang_hicache.go @@ -53,7 +53,7 @@ func (sglangHiCacheAdapter) SupportsBinding(binding *backendadapter.Binding) boo return binding == nil } -func (sglangHiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { +func (a sglangHiCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { if binding != nil { return fmt.Errorf("SGLang HiCache adapter does not support remote binding protocol %q", binding.Protocol) } diff --git a/internal/adapters/builtin/runtime/sglang_hicache_test.go b/internal/adapters/builtin/runtime/sglang_hicache_test.go index 3cb1d089..a98004af 100644 --- a/internal/adapters/builtin/runtime/sglang_hicache_test.go +++ b/internal/adapters/builtin/runtime/sglang_hicache_test.go @@ -123,6 +123,7 @@ func TestHiCacheOptionalFieldsStayOmitted(t *testing.T) { SGLangHiCacheWritePolicyArg, SGLangHiCacheIOBackendArg, SGLangHiCacheMemoryLayoutArg, + SGLangEnableMetricsArg, } { if _, ok := testArgValue(pod.Containers[0].Args, flag); ok { t.Errorf("unset optional field injected %s: %v", flag, pod.Containers[0].Args) diff --git a/internal/adapters/builtin/runtime/sglang_lmcache.go b/internal/adapters/builtin/runtime/sglang_lmcache.go index cb2a307a..245f0df8 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache.go @@ -16,13 +16,6 @@ import ( ) const ( - // sglangLMCacheMPConnectorProfile is the runtime-owner capability contract - // required before the webhook applies the typed PodLocal MP wire. The image - // pipeline owns this declaration; CacheBackend does not inspect or replace - // the engine image. - sglangLMCacheMPConnectorProfile = "sglang-lmcache-mp-v1" - sglangLMCacheMPClientVersion = "0.5.3" - // subscriberHashScheme is the canonical hash-scheme tag the SGLang // subscriber carries. Kept distinct from the runtime id and from vLLM's // "vllm" tag: the cache plane keys the index on (tenant, model, @@ -104,28 +97,39 @@ func (sglangLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) boo // InjectEngineConfig renders SGLang's LMCache MP-mode launch surface from a // host-only nil binding or a RESP binding for Redis L2 storage. -func (sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { +func (a sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { + var err error if cache != nil && cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" { - return injectSGLangLMCachePodLocal(pod, binding, cache) - } - endpoint := "" - if binding != nil { - if binding.Protocol != backendadapter.ProtocolRESP { - return fmt.Errorf("SGLang LMCache adapter does not support remote binding protocol %q", binding.Protocol) + err = injectSGLangLMCachePodLocal(pod, binding, cache) + } else { + endpoint := "" + if binding != nil { + if binding.Protocol != backendadapter.ProtocolRESP { + return fmt.Errorf("SGLang LMCache adapter does not support remote binding protocol %q", binding.Protocol) + } + endpoint = binding.Endpoint } - endpoint = binding.Endpoint + err = InjectSGLangLMCache(pod, endpoint, cache) + } + if err != nil { + return err } - return InjectSGLangLMCache(pod, endpoint, cache) + return ensureSGLangMetricsForSubscriber(pod, cache, a.subscriber) } -// ConnectorRequirement declares the engine-image-owned connector profile used -// by the typed SGLang MP adapter. Admission compares this with Pod annotations; -// it does not infer capability from an image name. -func (sglangLMCacheAdapter) ConnectorRequirement(*cachev1alpha1.CacheBackend) runtimeadapter.LMCacheConnectorRequirement { - return runtimeadapter.LMCacheConnectorRequirement{ - Profile: sglangLMCacheMPConnectorProfile, - ClientVersion: sglangLMCacheMPClientVersion, +// ensureSGLangMetricsForSubscriber makes the subscriber contract complete: +// SGLang does not expose /metrics unless --enable-metrics is present. Only add +// the flag when this CacheBackend will actually receive an observation sidecar. +func ensureSGLangMetricsForSubscriber(pod *corev1.PodSpec, cache *cachev1alpha1.CacheBackend, subscriber SubscriberConfig) error { + if subscriber.Image == "" || cache == nil || cache.Spec.EffectiveObservationModelID() == "" { + return nil } + engineIndex, err := EngineContainerIndexNamed(pod, SGLangEngineContainerName) + if err != nil { + return err + } + pod.Containers[engineIndex].Args = UpsertFlag(pod.Containers[engineIndex].Args, SGLangEnableMetricsArg) + return nil } // ValidateMPEnginePod checks the concrete Pod constraints needed before the diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index 2b12520e..33f064fd 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -231,14 +231,9 @@ func TestSGLangCanonicalHostOnlyBindingDoesNotSelectRedis(t *testing.T) { func TestSGLangTypedPodLocalUsesCommonRenderer(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) - mpAdapter, ok := adapter.(runtimeadapter.LMCacheMPRuntimeAdapter) - if !ok { + if _, ok := adapter.(runtimeadapter.LMCacheMPRuntimeAdapter); !ok { t.Fatalf("adapter %T does not implement LMCacheMPRuntimeAdapter", adapter) } - requirement := mpAdapter.ConnectorRequirement(newTypedSGLangMPBackend()) - if requirement.Profile != sglangLMCacheMPConnectorProfile || requirement.ClientVersion != "0.5.3" { - t.Fatalf("connector requirement = %+v", requirement) - } cache := newTypedSGLangMPBackend() pod := &corev1.PodSpec{Containers: []corev1.Container{{ @@ -277,22 +272,41 @@ func TestSGLangTypedPodLocalUsesCommonRenderer(t *testing.T) { } } +func TestSGLangInjectsMetricsOnlyWhenSubscriberWillAttach(t *testing.T) { + cache := newTypedSGLangMPBackend() + cache.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "gemma"} + pod := &corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, + }}} + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{Image: "subscriber:pinned"}) + if err := adapter.InjectEngineConfig(pod, nil, cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + if !containsArg(pod.Containers[0].Args, SGLangEnableMetricsArg) { + t.Fatalf("engine args missing %s required by subscriber: %v", SGLangEnableMetricsArg, pod.Containers[0].Args) + } + + withoutSubscriber := newTypedSGLangMPBackend() + pod = &corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, + }}} + if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, nil, withoutSubscriber); err != nil { + t.Fatalf("InjectEngineConfig without subscriber: %v", err) + } + if containsArg(pod.Containers[0].Args, SGLangEnableMetricsArg) { + t.Fatalf("engine args unexpectedly enabled metrics without a subscriber: %v", pod.Containers[0].Args) + } +} + func TestSGLangValidateTypedMPEnginePod(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) cache := newTypedSGLangMPBackend() pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ - runtimeadapter.AnnotationLMCacheConnectorProfile: sglangLMCacheMPConnectorProfile, - runtimeadapter.AnnotationLMCacheClientVersion: sglangLMCacheMPClientVersion, - }}, Spec: corev1.PodSpec{Containers: []corev1.Container{{ Name: SGLangEngineContainerName, Args: []string{"--page-size=1"}, }}}, } - if err := runtimeadapter.ValidateConnectorDeclaration(pod, adapter.ConnectorRequirement(cache)); err != nil { - t.Fatalf("ValidateConnectorDeclaration: %v", err) - } if err := adapter.ValidateMPEnginePod(pod, cache); err != nil { t.Fatalf("ValidateMPEnginePod: %v", err) } diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go index 243c8af6..064e3274 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go @@ -18,9 +18,6 @@ import ( ) const ( - vllmLMCacheMPConnectorProfile = "vllm-lmcache-mp-v1" - vllmLMCacheMPClientVersion = "0.5.3" - vllmLMCacheMPConnectorName = "LMCacheMPConnector" vllmLMCacheMPConnectorModulePath = "lmcache.integration.vllm.lmcache_mp_connector" vllmDisableHybridKVCacheArg = "--disable-hybrid-kv-cache-manager" @@ -53,13 +50,6 @@ func (vllmLMCacheMPAdapter) SupportsBinding(binding *backendadapter.Binding) boo return binding == nil || binding.Protocol == backendadapter.ProtocolRESP } -func (vllmLMCacheMPAdapter) ConnectorRequirement(*cachev1alpha1.CacheBackend) runtimeadapter.LMCacheConnectorRequirement { - return runtimeadapter.LMCacheConnectorRequirement{ - Profile: vllmLMCacheMPConnectorProfile, - ClientVersion: vllmLMCacheMPClientVersion, - } -} - // ValidateMPEnginePod rejects only constraints that can be classified from the // concrete Pod. The pinned LMCache connector fixes one MP server per vLLM // instance, and the initial production profile does not claim pipeline or @@ -85,17 +75,17 @@ func (vllmLMCacheMPAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a return err } args := pod.Spec.Containers[engineIndex].Args - if _, err := vllmPositiveParallelSize(args, []string{"--tensor-parallel-size", "-tp"}, 1); err != nil { + if _, err := positiveParallelSize(args, []string{"--tensor-parallel-size", "-tp"}, 1); err != nil { return fmt.Errorf("vLLM LMCache MP tensor parallelism: %w", err) } - pp, err := vllmPositiveParallelSize(args, []string{"--pipeline-parallel-size", "-pp"}, 1) + pp, err := positiveParallelSize(args, []string{"--pipeline-parallel-size", "-pp"}, 1) if err != nil { return fmt.Errorf("vLLM LMCache MP pipeline parallelism: %w", err) } if pp != 1 { return fmt.Errorf("vLLM LMCache MP pipeline parallel size %d is not supported by the initial PodLocal profile; use 1", pp) } - dp, err := vllmPositiveParallelSize(args, []string{"--data-parallel-size", "-dp"}, 1) + dp, err := positiveParallelSize(args, []string{"--data-parallel-size", "-dp"}, 1) if err != nil { return fmt.Errorf("vLLM LMCache MP data parallelism: %w", err) } @@ -123,7 +113,7 @@ func (vllmLMCacheMPAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a return nil } -func vllmPositiveParallelSize(args, flags []string, fallback int64) (int64, error) { +func positiveParallelSize(args, flags []string, fallback int64) (int64, error) { var values []string var seenFlags []string for index := 0; index < len(args); index++ { diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go index f55cd0ff..c9318cf6 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go @@ -77,14 +77,6 @@ func TestVLLMLMCacheMPRegistrySelectionDoesNotChangeLegacy(t *testing.T) { } } -func TestVLLMLMCacheMPConnectorRequirement(t *testing.T) { - adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) - requirement := adapter.ConnectorRequirement(newTypedVLLMMPBackend()) - if requirement.Profile != "vllm-lmcache-mp-v1" || requirement.ClientVersion != "0.5.3" { - t.Fatalf("connector requirement = %+v", requirement) - } -} - func TestVLLMLMCacheMPReservedSurface(t *testing.T) { adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) if got, want := adapter.ReservedArgs(), []string{defaultEngineKVTransferConfigArg, vllmDisableHybridKVCacheArg}; !reflect.DeepEqual(got, want) { diff --git a/internal/controller/cachebackend_kernelcheck.go b/internal/controller/cachebackend_kernelcheck.go index 94dfccfe..ecdb5d69 100644 --- a/internal/controller/cachebackend_kernelcheck.go +++ b/internal/controller/cachebackend_kernelcheck.go @@ -20,8 +20,8 @@ import ( "github.com/cachebox-project/inference-cache/internal/enginebinding" ) -// EngineKernelsHealthy gate: surfaces the engine-side native CUDA-kernel -// (lmcache c_ops) load health on the CacheBackend, read from the +// EngineKernelsHealthy gate: surfaces the engine-side native CUDA-extension +// (LMCache c_ops plus the vLLM native module) load health on the CacheBackend, read from the // lmcache-kernel-check init container the pod webhook injects into matched // engine pods. Default is fail-OPEN observability (report-only): the condition // surfaces the problem but does NOT downgrade Ready, so a degraded kernel tier @@ -38,7 +38,7 @@ const ( conditionTypeEngineKernelsHealthy = "EngineKernelsHealthy" reasonKernelsHealthy = "KernelsHealthy" - // reasonKernelLoadFailed covers every way the native lmcache c_ops + // reasonKernelLoadFailed covers every way the engine/LMCache native // kernels did not load: a libcudart/CUDA-runtime mismatch (the root // cause), a CPU/pure-python build with no compiled extension, or lmcache // not being importable at all. The specific cause is carried verbatim in @@ -111,7 +111,7 @@ func evaluateEngineKernelHealth( if cond.Status == metav1.ConditionFalse && strictFail { v.downgradeReady = true v.readyReason = reasonEngineKernelDegraded - v.readyMessage = "lmcache CUDA kernels failed to load on one or more engine pods; in strict mode those pods stay in Init holding their GPU reservation without serving — fix the engine image's lmcache/CUDA alignment or set " + enginebinding.AnnotationLMCacheKernelCheck + "=report-only" + v.readyMessage = "vLLM/LMCache CUDA extensions failed to load on one or more engine pods; in strict mode those pods stay in Init holding their GPU reservation without serving — fix the engine image's vLLM/LMCache/CUDA alignment or set " + enginebinding.AnnotationLMCacheKernelCheck + "=report-only" } return v } @@ -198,7 +198,7 @@ func aggregateKernelHealth(backend *cachev1alpha1.CacheBackend, pods []corev1.Po switch { case nFail > 0: return mk(metav1.ConditionFalse, reasonKernelLoadFailed, - fmt.Sprintf("native lmcache c_ops kernels failed to load on %d engine pod(s): %s", nFail, failMsg)), true, strictFail + fmt.Sprintf("native vLLM/LMCache CUDA extensions failed to load on %d engine pod(s): %s", nFail, failMsg)), true, strictFail case nErr > 0: return mk(metav1.ConditionUnknown, reasonKernelCheckError, fmt.Sprintf("kernel-check init container on %d engine pod(s) terminated without a recognized result message (%s)", nErr, errDetail)), true, strictFail @@ -207,7 +207,7 @@ func aggregateKernelHealth(backend *cachev1alpha1.CacheBackend, pods []corev1.Po fmt.Sprintf("kernel-check init container has not completed on %d engine pod(s)", nPending)), true, strictFail default: return mk(metav1.ConditionTrue, reasonKernelsHealthy, - fmt.Sprintf("native lmcache c_ops kernels loaded on %d engine pod(s)", nOK)), true, strictFail + fmt.Sprintf("native vLLM/LMCache CUDA extensions loaded on %d engine pod(s)", nOK)), true, strictFail } } diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index c94d1a00..3fc4d27e 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -25,7 +25,7 @@ const ( conditionTypeRemoteStorageReady = "RemoteStorageReady" reasonConnectorReady = "ConnectorReady" - reasonConnectorUnverified = "ConnectorCapabilityUnverified" + reasonConnectorUnverified = "ConnectorInjectionUnverified" reasonNoEnginePods = "NoEnginePods" reasonMPServersNotReady = "MPServersNotReady" reasonRemoteStorageReady = "RemoteStorageReady" @@ -128,7 +128,7 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con } else if verified != matched { status = metav1.ConditionUnknown reason = reasonConnectorUnverified - message = fmt.Sprintf("%d/%d selected engine Pods carry the webhook-authenticated connector declaration; unverified Pods are left un-injected", verified, matched) + message = fmt.Sprintf("%d/%d selected engine Pods carry the webhook-authenticated injection record for this CacheBackend generation; unverified Pods are left un-injected", verified, matched) } else if readyServers == desiredServers && readyEngines == matched { status = metav1.ConditionTrue reason = reasonConnectorReady diff --git a/internal/subscriber/events.go b/internal/subscriber/events.go index 300a90d0..a975c50d 100644 --- a/internal/subscriber/events.go +++ b/internal/subscriber/events.go @@ -14,11 +14,14 @@ import ( ) // This file decodes vLLM's KV-cache event stream. vLLM publishes an EventBatch -// per step as msgpack over ZMQ, using msgspec array-like tagged structs: +// per step as msgpack over ZMQ. The EventBatch envelope is array-like. Current +// vLLM releases encode nested KVCacheEvent structs as tagged maps, while SGLang +// and older vLLM-compatible publishers encode them as tagged tuples: // // EventBatch = [ts(float), events([...]), ...] // trailing fields (e.g. a // // data-parallel rank) are ignored -// event = [tag(string), ...fields] // tag is the struct name +// event = {"type": tag, ...fields} // vLLM 0.25+ +// | [tag(string), ...fields] // SGLang / legacy compatibility // BlockStored = ["BlockStored", block_hashes, parent_block_hash, token_ids, block_size, lora_id] // BlockRemoved = ["BlockRemoved", block_hashes] // AllBlocksCleared = ["AllBlocksCleared"] @@ -120,12 +123,18 @@ func DecodeEventBatch(payload []byte) (*EventBatch, error) { return out, nil } -// decodeEvent decodes a single [tag, ...fields] event. Returns (nil, nil) for an -// unknown tag so new vLLM event types don't break older subscribers. +// decodeEvent decodes either a current vLLM tagged map or a legacy/SGLang +// [tag, ...fields] tuple. Returns (nil, nil) for an unknown tag so new event +// types don't break older subscribers. func decodeEvent(raw msgpack.RawMessage) (Event, error) { + var keyed map[string]msgpack.RawMessage + if err := msgpack.Unmarshal(raw, &keyed); err == nil { + return decodeMapEvent(keyed) + } + var fields []msgpack.RawMessage if err := msgpack.Unmarshal(raw, &fields); err != nil { - return nil, fmt.Errorf("decode event tuple: %w", err) + return nil, fmt.Errorf("decode event: want tagged map or tuple: %w", err) } if len(fields) == 0 { return nil, fmt.Errorf("event tuple is empty") @@ -196,6 +205,89 @@ func decodeEvent(raw msgpack.RawMessage) (Event, error) { } } +func decodeMapEvent(fields map[string]msgpack.RawMessage) (Event, error) { + tagRaw, ok := fields["type"] + if !ok { + return nil, fmt.Errorf("event map has no type tag") + } + var tag string + if err := msgpack.Unmarshal(tagRaw, &tag); err != nil { + return nil, fmt.Errorf("decode event type: %w", err) + } + + switch tag { + case "BlockStored": + hashesRaw, ok := fields["block_hashes"] + if !ok { + return nil, fmt.Errorf("BlockStored.block_hashes is required") + } + hashes, err := decodeHashes(hashesRaw) + if err != nil { + return nil, fmt.Errorf("BlockStored.block_hashes: %w", err) + } + + var parent []byte + if parentRaw, exists := fields["parent_block_hash"]; exists { + parent, err = decodeParent(parentRaw) + if err != nil { + return nil, fmt.Errorf("BlockStored.parent_block_hash: %w", err) + } + } + + tokensRaw, ok := fields["token_ids"] + if !ok { + return nil, fmt.Errorf("BlockStored.token_ids is required") + } + tokenIDs, err := decodeTokenIDs(tokensRaw) + if err != nil { + return nil, fmt.Errorf("BlockStored.token_ids: %w", err) + } + + blockSizeRaw, ok := fields["block_size"] + if !ok { + return nil, fmt.Errorf("BlockStored.block_size is required") + } + var blockSize int32 + if err := msgpack.Unmarshal(blockSizeRaw, &blockSize); err != nil { + return nil, fmt.Errorf("BlockStored.block_size: %w", err) + } + if blockSize <= 0 { + return nil, fmt.Errorf("BlockStored.block_size must be positive, got %d", blockSize) + } + + var loraID *int64 + if loraRaw, exists := fields["lora_id"]; exists { + loraID, err = decodeLoRAID(loraRaw) + if err != nil { + return nil, fmt.Errorf("BlockStored.lora_id: %w", err) + } + } + return BlockStored{ + BlockHashes: hashes, + ParentBlockHash: parent, + TokenIDs: tokenIDs, + BlockSize: blockSize, + LoRAID: loraID, + }, nil + + case "BlockRemoved": + hashesRaw, ok := fields["block_hashes"] + if !ok { + return nil, fmt.Errorf("BlockRemoved.block_hashes is required") + } + hashes, err := decodeHashes(hashesRaw) + if err != nil { + return nil, fmt.Errorf("BlockRemoved.block_hashes: %w", err) + } + return BlockRemoved{BlockHashes: hashes}, nil + + case "AllBlocksCleared": + return AllBlocksCleared{}, nil + default: + return nil, nil + } +} + // decodeHashes decodes a msgpack array of block hashes into opaque bytes. Each // element is either binary (used as-is) or an integer (vLLM's int hash variant, // normalized to 8-byte big-endian). diff --git a/internal/subscriber/events_test.go b/internal/subscriber/events_test.go index 95d1c5b4..c3f8cf4a 100644 --- a/internal/subscriber/events_test.go +++ b/internal/subscriber/events_test.go @@ -7,6 +7,7 @@ package subscriber import ( "bytes" "encoding/binary" + "strings" "testing" "github.com/vmihailenco/msgpack/v5" @@ -28,6 +29,88 @@ func encodeVLLMBatch(t *testing.T, ts float64, events ...[]interface{}) []byte { return b } +func encodeVLLMMapBatch(t *testing.T, ts float64, events ...map[string]interface{}) []byte { + t.Helper() + evs := make([]interface{}, len(events)) + for i, event := range events { + evs[i] = event + } + b, err := msgpack.Marshal([]interface{}{ts, evs, 0}) + if err != nil { + t.Fatalf("encode map fixture: %v", err) + } + return b +} + +func TestDecodeVLLMTaggedMapEventBatch(t *testing.T) { + payload := encodeVLLMMapBatch(t, 1779901681.5, + map[string]interface{}{ + "type": "BlockStored", + "block_hashes": []uint64{10, 11}, + "parent_block_hash": uint64(7), + "token_ids": []int64{0, 1, 2, 3}, + "block_size": int32(128), + "lora_id": int64(4), + "medium": "GPU", + "group_idx": 0, + "kv_cache_spec_kind": "full", + "kv_cache_spec_sliding_window": nil, + "locality": "LOCAL", // vLLM 0.26 additive field + }, + map[string]interface{}{ + "type": "BlockRemoved", + "block_hashes": []uint64{10}, + "medium": "GPU", + "group_idx": 0, + "locality": "LOCAL", + }, + map[string]interface{}{"type": "AllBlocksCleared"}, + map[string]interface{}{"type": "SomeFutureEvent", "value": 42}, + ) + + batch, err := DecodeEventBatch(payload) + if err != nil { + t.Fatalf("DecodeEventBatch: %v", err) + } + if len(batch.Events) != 3 { + t.Fatalf("got %d events, want 3 (unknown skipped)", len(batch.Events)) + } + stored, ok := batch.Events[0].(BlockStored) + if !ok { + t.Fatalf("event[0] = %T, want BlockStored", batch.Events[0]) + } + if len(stored.BlockHashes) != 2 || binary.BigEndian.Uint64(stored.BlockHashes[1]) != 11 { + t.Errorf("BlockHashes = %v, want big-endian [10 11]", stored.BlockHashes) + } + if binary.BigEndian.Uint64(stored.ParentBlockHash) != 7 { + t.Errorf("ParentBlockHash = %x, want big-endian 7", stored.ParentBlockHash) + } + if stored.BlockSize != 128 || len(stored.TokenIDs) != 4 || stored.TokenIDs[3] != 3 { + t.Errorf("stored = %+v, want blockSize=128 and tokenIDs=[0 1 2 3]", stored) + } + if stored.LoRAID == nil || *stored.LoRAID != 4 { + t.Errorf("LoRAID = %v, want 4", stored.LoRAID) + } + if _, ok := batch.Events[1].(BlockRemoved); !ok { + t.Errorf("event[1] = %T, want BlockRemoved", batch.Events[1]) + } + if _, ok := batch.Events[2].(AllBlocksCleared); !ok { + t.Errorf("event[2] = %T, want AllBlocksCleared", batch.Events[2]) + } +} + +func TestDecodeVLLMTaggedMapRequiresBlockSize(t *testing.T) { + payload := encodeVLLMMapBatch(t, 1, map[string]interface{}{ + "type": "BlockStored", + "block_hashes": []uint64{1}, + "parent_block_hash": nil, + "token_ids": []int64{1, 2}, + }) + if _, err := DecodeEventBatch(payload); err == nil || !strings.Contains(err.Error(), "block_size is required") { + t.Fatalf("error = %v, want missing block_size", err) + } +} + func TestDecodeEventBatch(t *testing.T) { // A BlockStored carrying token_ids (now preserved for content hashing), a // BlockRemoved, an AllBlocksCleared, and an unknown event type (skipped). diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index 5e5d25e4..57f45080 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -320,10 +320,6 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { Name: "sglang-mp-engine", Namespace: ns, Labels: map[string]string{"app": "sglang-mp-test"}, - Annotations: map[string]string{ - "inferencecache.io/lmcache-connector-profile": "sglang-lmcache-mp-v1", - "inferencecache.io/lmcache-client-version": "0.5.3", - }, }, Spec: corev1.PodSpec{Containers: []corev1.Container{{ Name: "sglang", Image: "sglang:connector-ready", Args: []string{"--page-size=1"}, @@ -392,10 +388,6 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { Name: "vllm-mp-engine", Namespace: ns, Labels: map[string]string{"app": "vllm-mp-test"}, - Annotations: map[string]string{ - "inferencecache.io/lmcache-connector-profile": "vllm-lmcache-mp-v1", - "inferencecache.io/lmcache-client-version": "0.5.3", - }, }, Spec: corev1.PodSpec{Containers: []corev1.Container{{ Name: "vllm", Image: "vllm:connector-ready", Args: []string{"--model", "model-a", "--tensor-parallel-size=2"}, diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index 6ff31f2a..d8fabc82 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -202,12 +202,6 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi "runtime=%q adapter does not implement typed LMCache topology=%q (fail-open, no legacy injection)", runtimeID, cache.Spec.LMCache.Topology)) } - requirement := mpAdapter.ConnectorRequirement(cache) - if err := adapterruntime.ValidateConnectorDeclaration(&pod, requirement); err != nil { - log.V(1).Info("fail-open: engine connector capability is unverified", - "runtime", string(runtimeID), "error", err.Error()) - return failOpen(req, &pod, fmt.Sprintf("engine connector capability is unverified (fail-open): %v", err)) - } if err := mpAdapter.ValidateMPEnginePod(&pod, cache); err != nil { log.V(1).Info("fail-open: typed LMCache MP adapter rejected engine pod", "runtime", string(runtimeID), "error", err.Error()) diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index bdf71815..e0983216 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -345,7 +345,7 @@ func typedVLLMPodLocalBackend(name, namespace string, selector map[string]string return cb } -func TestHandle_TypedVLLMMissingCapabilityDoesNotFallThroughToLegacyAdapter(t *testing.T) { +func TestHandle_TypedVLLMWithoutCapabilityAnnotationsUsesDedicatedMPAdapter(t *testing.T) { const ns = "engines" cb := typedVLLMPodLocalBackend("primary", ns, map[string]string{"app": "vllm"}) h := newHandler(t, cb) @@ -353,14 +353,9 @@ func TestHandle_TypedVLLMMissingCapabilityDoesNotFallThroughToLegacyAdapter(t *t req := newRequest(t, pod, ns) resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("typed MP pod with unverified connector must fail open: %+v", resp.Result) - } - if len(resp.Patches) != 0 { - t.Fatalf("typed MP pod must not receive legacy injection; got %d patches", len(resp.Patches)) - } - if resp.Result == nil || !strings.Contains(resp.Result.Message, "engine connector capability is unverified") { - t.Fatalf("response message = %v, want connector-capability diagnostic", resp.Result) + if !resp.Allowed || len(resp.Patches) == 0 { + t.Fatalf("typed MP pod should be injected without capability annotations: allowed=%v patches=%d result=%+v", + resp.Allowed, len(resp.Patches), resp.Result) } } @@ -369,10 +364,6 @@ func TestHandle_TypedPodLocalVLLMUsesDedicatedMPAdapter(t *testing.T) { cb := typedVLLMPodLocalBackend("vllm-typed", ns, map[string]string{"app": "vllm-mp"}) h := newHandler(t, cb) pod := vllmEnginePod("engine-mp", map[string]string{"app": "vllm-mp"}) - pod.Annotations = map[string]string{ - adapterruntime.AnnotationLMCacheConnectorProfile: "vllm-lmcache-mp-v1", - adapterruntime.AnnotationLMCacheClientVersion: "0.5.3", - } pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--tensor-parallel-size=2") req := newRequest(t, pod, ns) @@ -517,11 +508,6 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { } h := newHandler(t, cb) pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) - if pod.Annotations == nil { - pod.Annotations = map[string]string{} - } - pod.Annotations[adapterruntime.AnnotationLMCacheConnectorProfile] = "sglang-lmcache-mp-v1" - pod.Annotations[adapterruntime.AnnotationLMCacheClientVersion] = "0.5.3" pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--page-size=1") req := newRequest(t, pod, ns) @@ -558,10 +544,6 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { // the engine wire. The response message is the actionable admission trace; // controller status subsequently counts the Pod as uncovered. incompatible := sglangEnginePod("sg-engine-incompatible", map[string]string{"app": "sglang"}) - incompatible.Annotations = map[string]string{ - adapterruntime.AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", - adapterruntime.AnnotationLMCacheClientVersion: "0.5.3", - } incompatible.Spec.Containers[0].Args = append(incompatible.Spec.Containers[0].Args, "--page-size=96") incompatibleReq := newRequest(t, incompatible, ns) incompatibleResp := h.Handle(context.Background(), incompatibleReq) diff --git a/internal/webhook/v1alpha1/cachebackend_integration_validation.go b/internal/webhook/v1alpha1/cachebackend_integration_validation.go index ac3dee17..6f1a5bb1 100644 --- a/internal/webhook/v1alpha1/cachebackend_integration_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_integration_validation.go @@ -174,17 +174,15 @@ func validHiCacheMemoryLayout(value cachev1alpha1.SGLangHiCacheMemoryLayout) boo } } -// rejectUnsupportedSGLangRole rejects a non-ReadWrite spec.integration.role on a -// (sglang, LMCache) backend. SGLang's --enable-lmcache integration has no -// kv_role split equivalent to vLLM's LMCache connector — it always both stores -// and retrieves — so a ReadOnly / WriteOnly role cannot be honored by the -// engine. Rejecting at admission makes that loud rather than silently treating -// the role as ReadWrite. Scoped to (sglang, LMCache): other engines map all -// roles onto their connector (vLLM), and the rule must not fire on an -// already-unsupported pair (e.g. sglang+External, which checkRuntimeAdapter -// rejects on its own). If SGLang's LMCache integration gains a -// producer/consumer split, lift this rule. -func rejectUnsupportedSGLangRole(cb *cachev1alpha1.CacheBackend) field.ErrorList { +// rejectUnsupportedLMCacheRole rejects a non-ReadWrite spec.integration.role +// on every LMCache backend. SGLang's integration has no producer/consumer +// split. vLLM accepts kv_consumer/kv_producer, but the validated LMCache 0.5.3 +// MP connector still stores as a consumer and retrieves as a producer. Until a +// pinned connector enforces directionality and passes GPU validation, accepting +// ReadOnly or WriteOnly would expose an API promise the data plane does not +// honor. Keep the rule scoped to LMCache so other backend implementations can +// define and validate their own role semantics independently. +func rejectUnsupportedLMCacheRole(cb *cachev1alpha1.CacheBackend) field.ErrorList { if cb.Spec.Integration == nil { return nil } @@ -192,15 +190,14 @@ func rejectUnsupportedSGLangRole(cb *cachev1alpha1.CacheBackend) field.ErrorList if role == "" || role == cachev1alpha1.CacheBackendIntegrationRoleReadWrite { return nil // unset defaults to ReadWrite; ReadWrite is honored } - if adapterruntime.ResolveRuntimeID(cb) != adapterruntime.RuntimeSGLang || - cb.Spec.Type != cachev1alpha1.CacheBackendTypeLMCache { + if cb.Spec.Type != cachev1alpha1.CacheBackendTypeLMCache { return nil } return field.ErrorList{ field.Invalid( field.NewPath("spec", "integration", "role"), role, - fmt.Sprintf("the sglang engine's LMCache integration has no producer/consumer split, so only %q (the default) is honored today; %q / %q are not yet wired for SGLang", + fmt.Sprintf("LMCache integrations currently support only %q (the default); %q / %q are rejected until a validated connector enforces directional cache access", cachev1alpha1.CacheBackendIntegrationRoleReadWrite, cachev1alpha1.CacheBackendIntegrationRoleReadOnly, cachev1alpha1.CacheBackendIntegrationRoleWriteOnly), diff --git a/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go b/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go index 8a2adc46..935a193b 100644 --- a/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go @@ -823,40 +823,47 @@ func TestValidator_RuntimeAdapter_VLLMPlusMooncakeAdmittedViaShippingRegistry(t } } -func TestValidator_SGLangRoleRejected(t *testing.T) { - // SGLang's LMCache integration has no producer/consumer split, so a - // non-ReadWrite role can't be honored — admission rejects it loudly rather - // than silently treating it as ReadWrite. Uses the real shipping registry - // (the role rule is registry-independent, but this keeps the adapter-pair - // check happy for the (sglang, LMCache) CR). +func TestValidator_LMCacheDirectionalRolesRejected(t *testing.T) { + // Neither shipping LMCache integration can currently honor directional + // roles: SGLang has no split, while the validated vLLM MP connector ignores + // kv_consumer/kv_producer behaviorally. v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - for _, role := range []cachev1alpha1.CacheBackendIntegrationRole{ - cachev1alpha1.CacheBackendIntegrationRoleReadOnly, - cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, + for _, runtime := range []cachev1alpha1.CacheBackendRuntime{ + cachev1alpha1.CacheBackendRuntimeVLLM, + cachev1alpha1.CacheBackendRuntimeSGLang, } { - t.Run(string(role), func(t *testing.T) { - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{Role: role} - requireInvalidWithCause(t, v, cb, "spec.integration.role", "sglang") - }) + for _, role := range []cachev1alpha1.CacheBackendIntegrationRole{ + cachev1alpha1.CacheBackendIntegrationRoleReadOnly, + cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, + } { + t.Run(string(runtime)+"/"+string(role), func(t *testing.T) { + cb := newBackend() // type=LMCache + cb.Spec.Runtime = runtime + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{Role: role} + requireInvalidWithCause(t, v, cb, "spec.integration.role", "directional cache access") + }) + } } } -func TestValidator_SGLangRoleReadWriteAndUnsetAdmitted(t *testing.T) { - // ReadWrite (and unset, which defaults to ReadWrite) are the honored - // SGLang role; both must admit. +func TestValidator_LMCacheReadWriteAndUnsetAdmitted(t *testing.T) { + // ReadWrite (and unset, which defaults to ReadWrite) are the only currently + // honored LMCache roles for both engines. v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cases := []*cachev1alpha1.CacheBackendIntegrationSpec{ - {}, // role unset - {Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, - } - for _, integ := range cases { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = integ - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("sglang role=%q rejected: %v", integ.Role, err) + for _, runtime := range []cachev1alpha1.CacheBackendRuntime{ + cachev1alpha1.CacheBackendRuntimeVLLM, + cachev1alpha1.CacheBackendRuntimeSGLang, + } { + for _, integ := range []*cachev1alpha1.CacheBackendIntegrationSpec{ + {}, // role unset + {Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, + } { + cb := newBackend() + cb.Spec.Runtime = runtime + cb.Spec.Integration = integ + if _, err := v.ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("runtime=%q role=%q rejected: %v", runtime, integ.Role, err) + } } } } diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index 5dd97875..d42ad589 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -17,8 +17,9 @@ import ( ) const ( - lmcacheKVEventPort int32 = 5557 - lmcacheMPHTTPPort int32 = 8080 + lmcacheKVEventPort int32 = 5557 + lmcacheMPHTTPPort int32 = 8080 + lmcacheMPMemoryHeadroom = "1Gi" ) var sha256ImagePattern = regexp.MustCompile(`^[^[:space:]@]+@sha256:[a-f0-9]{64}$`) @@ -205,13 +206,20 @@ func validateMPServer( errs = append(errs, field.Required(path.Child("resources", "requests").Key(string(corev1.ResourceCPU)), "a positive CPU request is required for the MP server")) } + var memoryBudget *resource.Quantity + if l1Capacity != nil && l1Capacity.Sign() > 0 { + budget := l1Capacity.DeepCopy() + budget.Add(resource.MustParse(lmcacheMPMemoryHeadroom)) + memoryBudget = &budget + } + memoryRequest, hasMemoryRequest := resources.Requests[corev1.ResourceMemory] if !hasMemoryRequest || memoryRequest.Sign() <= 0 { errs = append(errs, field.Required(path.Child("resources", "requests").Key(string(corev1.ResourceMemory)), "a positive memory request is required for the MP server")) - } else if l1Capacity != nil && l1Capacity.Sign() > 0 && memoryRequest.Cmp(*l1Capacity) <= 0 { + } else if memoryBudget != nil && memoryRequest.Cmp(*memoryBudget) < 0 { errs = append(errs, field.Invalid(path.Child("resources", "requests").Key(string(corev1.ResourceMemory)), - memoryRequest.String(), fmt.Sprintf("must be greater than l1Capacity %s so the server has explicit memory headroom", l1Capacity.String()))) + memoryRequest.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) so scheduling accounts for the memory-backed /dev/shm", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) } memoryLimit, hasMemoryLimit := resources.Limits[corev1.ResourceMemory] @@ -223,9 +231,9 @@ func validateMPServer( errs = append(errs, field.Invalid(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), memoryLimit.String(), fmt.Sprintf("must be greater than or equal to the memory request %s", memoryRequest.String()))) } - if l1Capacity != nil && l1Capacity.Sign() > 0 && memoryLimit.Cmp(*l1Capacity) <= 0 { + if memoryBudget != nil && memoryLimit.Cmp(*memoryBudget) < 0 { errs = append(errs, field.Invalid(path.Child("resources", "limits").Key(string(corev1.ResourceMemory)), - memoryLimit.String(), fmt.Sprintf("must be greater than l1Capacity %s so the server has explicit memory headroom", l1Capacity.String()))) + memoryLimit.String(), fmt.Sprintf("must be at least %s (l1Capacity %s + %s headroom) to bound the memory-backed /dev/shm", memoryBudget.String(), l1Capacity.String(), lmcacheMPMemoryHeadroom))) } } diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index 3c4ace41..eb2c94c1 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -218,12 +218,26 @@ func TestValidateLMCacheTopology(t *testing.T) { wantField: "spec.lmCache.podLocal.server.port", }, { - name: "memory request has no headroom", + name: "memory request below shm budget", mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Gi") + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1536Mi") }, wantField: "spec.lmCache.podLocal.server.resources.requests[memory]", }, + { + name: "memory limit below shm budget", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("1536Mi") + }, + wantField: "spec.lmCache.podLocal.server.resources.limits[memory]", + }, + { + name: "memory request and limit equal shm budget", + mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("2Gi") + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("2Gi") + }, + }, { name: "fractional extended resource", mutate: func(cb *cachev1alpha1.CacheBackend) { diff --git a/internal/webhook/v1alpha1/cachebackend_validator.go b/internal/webhook/v1alpha1/cachebackend_validator.go index 4cca642a..6f8491f6 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator.go +++ b/internal/webhook/v1alpha1/cachebackend_validator.go @@ -67,7 +67,7 @@ var DefaultValidationRules = []ValidationRule{ rejectEventsOnlyMisconfiguration, validateSGLangHiCache, rejectInvalidKernelCheckAnnotation, - rejectUnsupportedSGLangRole, + rejectUnsupportedLMCacheRole, rejectSGLangRedisL2ScaleOut, } diff --git a/internal/webhook/v1alpha1/cachebackend_validator_test.go b/internal/webhook/v1alpha1/cachebackend_validator_test.go index ebd70136..97792513 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator_test.go +++ b/internal/webhook/v1alpha1/cachebackend_validator_test.go @@ -390,18 +390,15 @@ func withSGLangOverrides(o cachev1alpha1.EngineInjectionOverrides) *cachev1alpha return cb } -func TestValidator_VLLMRoleReadOnlyStillAdmitted(t *testing.T) { - // The SGLang role rule must not bleed onto vLLM: vLLM maps ReadOnly onto - // its connector (kv_consumer), so a (vllm, LMCache) backend with ReadOnly - // must still admit. +func TestValidator_VLLMRoleReadOnlyRejected(t *testing.T) { + // vLLM renders ReadOnly as kv_consumer, but LMCache 0.5.3 does not enforce + // that directionality. Admission must reject the unsupported API promise. v := &CacheBackendValidator{Registry: defaultShippingRegistry()} cb := newBackend() cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ Role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, } - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("vllm role=ReadOnly rejected by the sglang role rule: %v", err) - } + requireInvalidWithCause(t, v, cb, "spec.integration.role", "directional cache access") } // eventsOnlyIntegration returns an integration spec wired for the events-only diff --git a/pkg/adapters/runtime/adapter.go b/pkg/adapters/runtime/adapter.go index 275784d7..1dd6b87e 100644 --- a/pkg/adapters/runtime/adapter.go +++ b/pkg/adapters/runtime/adapter.go @@ -111,24 +111,6 @@ type KVCacheRuntimeAdapter interface { EngineContainerName() string } -const ( - // AnnotationLMCacheConnectorProfile is the runtime-owner declaration of the - // engine-side LMCache connector API implemented by the workload image. - AnnotationLMCacheConnectorProfile = "inferencecache.io/lmcache-connector-profile" - // AnnotationLMCacheClientVersion declares the LMCache client package version - // validated by the runtime owner's image pipeline. - AnnotationLMCacheClientVersion = "inferencecache.io/lmcache-client-version" -) - -// LMCacheConnectorRequirement is the capability contract an MP runtime adapter -// requires from an inference workload. It names an interface profile rather -// than an image, so any inference system can provide a compatible image without -// CacheBackend owning or allowlisting that image. -type LMCacheConnectorRequirement struct { - Profile string - ClientVersion string -} - // LMCacheMPRuntimeAdapter is the Phase-1 gate for adapters that understand the // final typed LMCache topology. Legacy adapters intentionally do not implement // it: the Pod webhook then admits a new MP Pod unmodified instead of silently @@ -137,34 +119,12 @@ type LMCacheConnectorRequirement struct { type LMCacheMPRuntimeAdapter interface { KVCacheRuntimeAdapter - // ConnectorRequirement returns the workload-owned capability declaration - // required by this adapter. - ConnectorRequirement(*cachev1alpha1.CacheBackend) LMCacheConnectorRequirement - // ValidateMPEnginePod validates version/parallelism/command/resource // constraints visible only on the concrete engine Pod. An unclassifiable // Pod returns an error and is never silently injected. ValidateMPEnginePod(*corev1.Pod, *cachev1alpha1.CacheBackend) error } -// ValidateConnectorDeclaration compares the runtime owner's Pod annotations -// with an adapter's required connector contract. This is deliberately a -// declaration check, not registry/image introspection; build-time probes and -// digest pinning bind the claim to image contents outside the admission path. -func ValidateConnectorDeclaration(pod *corev1.Pod, requirement LMCacheConnectorRequirement) error { - if pod == nil { - return fmt.Errorf("engine pod is nil") - } - annotations := pod.GetAnnotations() - if got := annotations[AnnotationLMCacheConnectorProfile]; got != requirement.Profile { - return fmt.Errorf("annotation %s=%q, want %q", AnnotationLMCacheConnectorProfile, got, requirement.Profile) - } - if got := annotations[AnnotationLMCacheClientVersion]; got != requirement.ClientVersion { - return fmt.Errorf("annotation %s=%q, want %q", AnnotationLMCacheClientVersion, got, requirement.ClientVersion) - } - return nil -} - // ErrNoAdapter is returned by [Registry.Select] when no registered adapter // supports a given (runtime, CacheBackend) pair. An admission validator can // translate this into a user-visible rejection; the reconciler logs and skips. diff --git a/pkg/adapters/runtime/adapter_test.go b/pkg/adapters/runtime/adapter_test.go index 0610cb16..065e525b 100644 --- a/pkg/adapters/runtime/adapter_test.go +++ b/pkg/adapters/runtime/adapter_test.go @@ -344,58 +344,6 @@ func TestResolveRuntimeID(t *testing.T) { } } -func TestValidateConnectorDeclaration(t *testing.T) { - requirement := LMCacheConnectorRequirement{ - Profile: "sglang-lmcache-mp-v1", - ClientVersion: "0.5.3", - } - tests := []struct { - name string - annotations map[string]string - wantErr bool - }{ - { - name: "matching declaration", - annotations: map[string]string{ - AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", - AnnotationLMCacheClientVersion: "0.5.3", - }, - }, - {name: "missing declaration", wantErr: true}, - { - name: "profile mismatch", - annotations: map[string]string{ - AnnotationLMCacheConnectorProfile: "vllm-lmcache-mp-v1", - AnnotationLMCacheClientVersion: "0.5.3", - }, - wantErr: true, - }, - { - name: "version mismatch", - annotations: map[string]string{ - AnnotationLMCacheConnectorProfile: "sglang-lmcache-mp-v1", - AnnotationLMCacheClientVersion: "0.5.2", - }, - wantErr: true, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Annotations: tc.annotations}} - err := ValidateConnectorDeclaration(pod, requirement) - if tc.wantErr && err == nil { - t.Fatal("expected declaration error") - } - if !tc.wantErr && err != nil { - t.Fatalf("unexpected declaration error: %v", err) - } - }) - } - if err := ValidateConnectorDeclaration(nil, requirement); err == nil { - t.Fatal("nil pod should be rejected") - } -} - func lookupEnv(env []corev1.EnvVar, name string) (string, bool) { for _, e := range env { if e.Name == name { diff --git a/test/fixtures/sglang-lmcache/Dockerfile b/test/fixtures/sglang-lmcache/Dockerfile index ad546875..955de757 100644 --- a/test/fixtures/sglang-lmcache/Dockerfile +++ b/test/fixtures/sglang-lmcache/Dockerfile @@ -13,5 +13,4 @@ RUN python3 -m pip install --no-cache-dir "lmcache==0.5.3" \ LABEL org.opencontainers.image.title="inference-cache SGLang LMCache connector fixture" \ org.opencontainers.image.description="SGLang v0.5.13.post1-cu130 with LMCache 0.5.3 for PodLocal MP validation" \ - io.inferencecache.lmcache-connector-profile="sglang-lmcache-mp-v1" \ io.inferencecache.lmcache-client-version="0.5.3" From 22baaa96ac6e6b366772cd1e246ec9e01cb28318 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Tue, 11 Aug 2026 12:33:57 -0700 Subject: [PATCH 06/13] Migrate repository consumers to LMCache MP Signed-off-by: Yue Sun --- .github/workflows/c2-reconciler-canary.yml | 12 +- .github/workflows/c6-engine-wiring-canary.yml | 16 +- .github/workflows/default-install-smoke.yml | 20 +- README.md | 13 +- config/samples/README.md | 32 +- .../samples/cache_v1alpha1_cachebackend.yaml | 28 +- config/samples/cachebackend-cpu-override.yaml | 48 +-- config/samples/cachebackend-external.yaml | 68 ++- config/samples/cachebackend-lmcache-cpu.yaml | 48 --- config/samples/cachebackend-lmcache.yaml | 64 ++- config/samples/cachebackend-mooncake.yaml | 107 ----- .../cachebackend-sglang-host-only.yaml | 30 +- config/samples/cachebackend-sglang.yaml | 77 +--- config/samples/cachebackend-with-engine.yaml | 111 ++--- .../samples/cachebackend-with-override.yaml | 81 ++-- config/samples/recipe-cpu-dev.yaml | 47 ++- config/samples/recipe-external-cache.yaml | 43 +- config/samples/recipe-gpu-production.yaml | 72 ++-- config/samples/recipe-multi-tenant.yaml | 45 +- config/samples/recipe-tuning.yaml | 39 +- docs/concepts/cachebackend-engine-binding.md | 151 ++++--- .../concepts/cachebackend-engine-overrides.md | 327 ++++----------- docs/design/cachebackend-api.md | 149 ++++--- docs/design/kvevent-subscriber-wiring.md | 26 +- .../lmcache-multiprocess-migration-roadmap.md | 73 +++- docs/design/lmcache-server-persistence.md | 12 +- docs/design/sglang-lmcache-mp-mode.md | 8 + docs/quickstart.md | 73 ++-- docs/reference-stack/GPU-RUNBOOK.md | 16 +- docs/reference-stack/README.md | 267 ++++-------- docs/reference-stack/VERSIONS.md | 175 ++------ .../helm/values-reference.yaml | 60 --- .../reference-stack/manifests/deployment.yaml | 53 ++- .../manifests/sglang-lmcache/README.md | 386 ++++-------------- .../manifests/sglang-lmcache/deployment.yaml | 193 +++------ .../scripts/canary_c2_reconcile.sh | 6 +- .../scripts/canary_c6_engine_wiring.sh | 6 +- .../scripts/default_install_smoke.sh | 134 ++++-- site/content/en/docs/concepts/cachebackend.md | 214 +++------- site/content/en/docs/concepts/pdtopology.md | 2 +- site/content/en/docs/overview/_index.md | 7 +- site/content/en/docs/reference/crd-api.md | 11 +- site/content/en/docs/tasks/bind-an-engine.md | 176 ++------ .../en/docs/tasks/deploy-a-cache-backend.md | 49 ++- 44 files changed, 1317 insertions(+), 2258 deletions(-) delete mode 100644 config/samples/cachebackend-lmcache-cpu.yaml delete mode 100644 config/samples/cachebackend-mooncake.yaml delete mode 100644 docs/reference-stack/helm/values-reference.yaml diff --git a/.github/workflows/c2-reconciler-canary.yml b/.github/workflows/c2-reconciler-canary.yml index cf04d921..83360b22 100644 --- a/.github/workflows/c2-reconciler-canary.yml +++ b/.github/workflows/c2-reconciler-canary.yml @@ -2,7 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 -# On-demand / scheduled CPU canary for the C2 CacheBackend reconciler. +# Manual legacy-IP compatibility canary for the C2 CacheBackend reconciler. +# It intentionally exercises remoteStorage.provider=LMCacheServer until Phase 7; +# it is not a current deployment reference and no longer runs on a schedule. # # Runs docs/reference-stack/scripts/canary_c2_reconcile.sh: brings up a kind # cluster, runs the controller, applies a CPU-profile CacheBackend, and asserts @@ -13,8 +15,8 @@ # GPU-free. # # This is NOT a per-PR gate (it pulls a multi-GB image, needs Docker + kind, and -# ~10 GiB RAM); it runs on a schedule and on manual dispatch. -name: c2-reconciler-canary +# ~10 GiB RAM). +name: legacy-ip-c2-reconciler-canary on: workflow_dispatch: @@ -23,14 +25,12 @@ on: description: "Runner label (override to target a self-hosted Docker host)" default: ubuntu-latest required: false - schedule: - - cron: "30 7 * * *" # nightly 07:30 UTC permissions: contents: read concurrency: - group: c2-reconciler-canary + group: legacy-ip-c2-reconciler-canary cancel-in-progress: false jobs: diff --git a/.github/workflows/c6-engine-wiring-canary.yml b/.github/workflows/c6-engine-wiring-canary.yml index 1d8fb00a..836f648b 100644 --- a/.github/workflows/c6-engine-wiring-canary.yml +++ b/.github/workflows/c6-engine-wiring-canary.yml @@ -2,8 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -# On-demand / scheduled CPU canary for the C6 vLLM+LMCache engine-pod -# wiring webhook + cross-pod cache reuse. +# Manual legacy-IP compatibility canary for the C6 vLLM+LMCache engine-pod +# wiring webhook + cross-pod cache reuse. It intentionally exercises +# LMCacheConnectorV1 and LMCacheServer until Phase 7; it is not a current +# deployment reference and no longer runs on a schedule. # # Runs docs/reference-stack/scripts/canary_c6_engine_wiring.sh: brings up # a kind cluster, installs cert-manager + this repo's config/default @@ -15,10 +17,8 @@ # prompt prefix engine-a previously serviced via the shared lmcache-server. # # This is NOT a per-PR gate (multi-GB image pull, ~12 GiB RAM for two CPU -# engines + lmcache-server + cert-manager); it runs on a schedule and on -# manual dispatch. Time-offset 1h after the C2 reconciler canary so two -# kind clusters never share a runner. -name: c6-engine-wiring-canary +# engines + lmcache-server + cert-manager). +name: legacy-ip-c6-engine-wiring-canary on: workflow_dispatch: @@ -31,14 +31,12 @@ on: description: "Skip the traffic-driving step (still asserts webhook wiring)" default: "0" required: false - schedule: - - cron: "30 8 * * *" # nightly 08:30 UTC, offset from C2 canary at 07:30 permissions: contents: read concurrency: - group: c6-engine-wiring-canary + group: legacy-ip-c6-engine-wiring-canary cancel-in-progress: false jobs: diff --git a/.github/workflows/default-install-smoke.yml b/.github/workflows/default-install-smoke.yml index 8fbe8570..d67b9c06 100644 --- a/.github/workflows/default-install-smoke.yml +++ b/.github/workflows/default-install-smoke.yml @@ -12,24 +12,16 @@ # - cacheindex/cluster-default.status.observedServer is populated (proves the # controller's CacheIndex poller is talking to the server's /snapshot) # - gRPC LookupRoute returns reason_code=NO_HINT (fail-open default) -# - paired sample (config/samples/cachebackend-with-engine.yaml) wires the -# CacheBackend ↔ engine-pod binding: status.matchedEnginePods=1, the -# injected-by annotation is stamped on the engine pod, and an -# InjectedByCacheBackend Event lands on the persisted pod's UID; then -# scaling the engine to 0 drives status.matchedEnginePods=0 via the -# reconciler's RequeueAfter cadence (no CR-side change). -# - External CacheBackend end-to-end: type=External renders no -# Deployment/Service, status.endpoint mirrors spec.endpoint, Ready=True, -# a matching engine pod is admitted with LMCACHE_REMOTE_URL injected -# from the operator-supplied endpoint, and admission rejects the -# known-bad shapes (non-lm:// scheme, empty host, non-External + endpoint) +# - typed PodLocal MP injection for vLLM and SGLang, including native sidecar, +# connector arguments, shared memory, and optional Redis L3 rendering +# - explicitly labelled legacy-IP compatibility checks for the implementation +# retained until Phase 7; these inline fixtures are not deployment examples # # Lightweight (two distroless ~30 MB images + cert-manager + a busybox # stand-in for the engine container + a pause-image pod, no real engine # pull), so it runs on every PR. Sister-canaries -# (c2-reconciler-canary, c6-engine-wiring-canary, cpu-substrate-canary) -# cover real engine pods + cross-pod cache reuse with multi-GB images; -# they stay schedule-only. +# (legacy-ip-c2-reconciler-canary, legacy-ip-c6-engine-wiring-canary) remain +# manual compatibility checks until Phase 7. name: default-install-smoke on: diff --git a/README.md b/README.md index bd61e3e9..00b002f5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A **vendor-neutral, Kubernetes-native cache-policy control plane for LLM inferen inference-cache makes routing **cache-aware**: it tracks which replica already holds a prompt's prefix warm and returns that as a routing *hint*, so a gateway can reuse KV/prefill instead of recomputing it — cutting time-to-first-token and cost. It -**orchestrates** existing KV-cache technology (LMCache, Mooncake); it is **not** a new +**orchestrates** existing KV-cache technology (LMCache); it is **not** a new distributed cache and **not** the data-plane gateway. Guiding principle — **"we decide routing; the gateway follows"**: all routing intelligence lives in the server, and the gateway simply tokenizes → calls `LookupRoute` → routes to the returned replica → @@ -27,16 +27,17 @@ The API separates three choices: `spec.runtime` selects the inference runtime, provider. Supporting another combination is an adapter addition; the core gRPC contract stays stable. -- **vLLM + LMCache** supports host-only caching, a managed or external - `LMCacheServer`, and managed or external `Mooncake` storage. -- **SGLang + LMCache** supports host-only caching or a managed/external Redis - remote store. +- **vLLM + LMCache** supports typed PodLocal MP with host-only caching or an + optional managed/external Redis L3. +- **SGLang + LMCache** supports the same typed PodLocal MP and Redis profiles; + the engine launch surface remains SGLang-specific. - **SGLang + SGLangHiCache** uses SGLang's native host tier and does not accept a remote-storage binding. See [`config/samples/`](config/samples/) for canonical manifests and [`docs/design/cachebackend-api.md`](docs/design/cachebackend-api.md) for the -compatibility rules retained for older v1alpha1 resources. +current contract and clearly labeled legacy-compatibility sections retained +until Phase 7 of the migration. ## What's Inference Cache? diff --git a/config/samples/README.md b/config/samples/README.md index 1372a572..61aeff2b 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -22,38 +22,40 @@ multi-tenant, Namespaces): equivalent typed vLLM PodLocal profiles for [host-only](cachebackend-vllm-podlocal-host-only.yaml), [managed Redis](cachebackend-vllm-podlocal-managed-redis.yaml), and - [external Redis](cachebackend-vllm-podlocal-external-redis.yaml). The - `recipe-*.yaml` catalog remains the maintained entry point for legacy - in-process LMCache scenarios until the repository-wide Phase-5 migration. + [external Redis](cachebackend-vllm-podlocal-external-redis.yaml). All LMCache + offload samples use the typed PodLocal MP API; `EventsOnly` intentionally + carries no LMCache data plane. ## Recipe catalog Each recipe is a single file with a top-of-file comment explaining the scenario and the apply steps. Most are self-contained; see "Prerequisites per recipe" below for the two that aren't (external cache, multi-tenant), and note -`recipe-gpu-production` is a shape template whose placeholder images you pin -before applying. All but `recipe-gpu-production` run without a GPU. +`recipe-gpu-production` is a shape template whose engine image you pin before +applying. Admission and sample validation require no GPU; actual LMCache MP +startup requires a compatible engine connector/package and the selected +runtime hardware. | Recipe | Use case | | --- | --- | -| [`recipe-cpu-dev.yaml`](recipe-cpu-dev.yaml) | Fastest path on a laptop / kind — tiny ungated model, no GPU, single replica, no quotas. | -| [`recipe-gpu-production.yaml`](recipe-gpu-production.yaml) | Typical production — real model on GPU engine pods, managed-backend autoscaling, a CachePolicy with production TTLs. | -| [`recipe-external-cache.yaml`](recipe-external-cache.yaml) | External `LMCacheServer` ownership — point the operator at a cache server you manage yourself; the controller provisions nothing. | +| [`recipe-cpu-dev.yaml`](recipe-cpu-dev.yaml) | Small single-replica typed-MP binding shape; engine startup still requires a connector-compatible image. | +| [`recipe-gpu-production.yaml`](recipe-gpu-production.yaml) | Production shape — GPU engine Pods, per-Pod MP L1, explicit managed Redis L3, and a production CachePolicy. | +| [`recipe-external-cache.yaml`](recipe-external-cache.yaml) | Typed MP with external Redis L3; the controller provisions no remote provider. | | [`recipe-multi-tenant.yaml`](recipe-multi-tenant.yaml) | Two CacheTenants + two CacheBackends across two namespaces — isolated cache identity and entry-count quotas; separate engines for per-tenant memory isolation. | -| [`recipe-tuning.yaml`](recipe-tuning.yaml) | CPU-dev shape plus a meaningful `engineOverrides` block (tune `LMCACHE_CHUNK_SIZE`, add `LMCACHE_LOG_LEVEL=DEBUG`). | +| [`recipe-tuning.yaml`](recipe-tuning.yaml) | Small typed-MP shape: typed `chunkSizeTokens` plus an `engineOverrides` log-level addition. | **Prerequisites per recipe.** Most recipes are self-contained. One has an -external dependency: `recipe-external-cache.yaml` needs a cache server already +external dependency: `recipe-external-cache.yaml` needs Redis already running at the endpoint you supply (replace the placeholder). `recipe-multi-tenant.yaml` has no external dependency but creates and deploys into two namespaces of its own. **Apply + observability.** Each recipe's `kubectl apply` wires matching engine -pods to the cache. For *managed* backends the wiring becomes available once the -controller publishes `status.endpoint`, so a pod admitted before then races past -injection and runs unwired until recreated (see each recipe's header); externally -owned backends wire straight from `spec.remoteStorage.endpoint` and have no such -race. KV reuse then works, but a *managed* backend only reaches `Ready=True` +pods to the cache. Host-only PodLocal wiring needs no provider endpoint. A +managed Redis L3 is controller-resolved, so apply the CacheBackend before +creating engine Pods; externally owned Redis uses the declared endpoint. KV +reuse then works when the runtime-owned image is compatible, but a managed +backend only reaches `Ready=True` and reports index entries once the `kvevent-subscriber` sidecar is auto-attached, which requires the controller to run with `--kvevent-subscriber-image` set (empty by default); otherwise it holds at `AwaitingFirstKVEvent` and then diff --git a/config/samples/cache_v1alpha1_cachebackend.yaml b/config/samples/cache_v1alpha1_cachebackend.yaml index 39dcd0a4..5d5a1d52 100644 --- a/config/samples/cache_v1alpha1_cachebackend.yaml +++ b/config/samples/cache_v1alpha1_cachebackend.yaml @@ -6,10 +6,11 @@ # applies cleanly against the current schema + admission webhook. Hand-curated, # scenario-specific recipes live in the sibling cachebackend-*.yaml files. # -# The runtime, engine cache, and remote provider are explicit so this minimum -# sample also demonstrates the canonical API hierarchy. Admission still -# defaults deploymentKind=Deployment, replicas=1, integration.role=ReadWrite, -# integration.failOpen=true, and observation.firstEventTimeout=5m. +# The runtime and typed PodLocal LMCache topology are explicit. Omitting +# remoteStorage intentionally selects host-only MP; no provider workload or +# cross-Pod sharing is implied. Admission still defaults integration.role to +# ReadWrite, integration.failOpen to true, and observation.firstEventTimeout to +# 5m. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -23,9 +24,20 @@ spec: engineSelector: matchLabels: inferencecache.io/cache-enabled: "true" + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: modelID: meta-llama/Meta-Llama-3-8B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: {} diff --git a/config/samples/cachebackend-cpu-override.yaml b/config/samples/cachebackend-cpu-override.yaml index 58904ee0..2b782dbc 100644 --- a/config/samples/cachebackend-cpu-override.yaml +++ b/config/samples/cachebackend-cpu-override.yaml @@ -2,20 +2,17 @@ # # SPDX-License-Identifier: Apache-2.0 -# CacheBackend for a CPU-vLLM dev cluster (no GPU) that uses -# spec.integration.engineOverrides to tune the engine container the pod -# webhook injects. +# Typed vLLM PodLocal LMCache MP example that uses +# spec.integration.engineOverrides to amend the inference-owner's engine +# container. The manifest is admission-valid without a GPU; actual engine +# startup remains the authoritative connector/package compatibility check. # # The override surface is engine-agnostic K8s vocabulary (args + env), so it # also extends to future runtime adapters and remote bindings with no CRD churn. # -# Admission HARD-REJECTS overrides that overlap the adapter's reserved -# args/env — for the vLLM+LMCache adapter today: `--kv-transfer-config`, -# `VLLM_USE_V1`, `LMCACHE_REMOTE_URL`, `INFERENCECACHE_FAIL_OPEN`. Suppressing -# any of those would un-wire the LMCache integration itself; if you don't -# want LMCache at all, use the per-pod `inferencecache.io/skip-inject` -# annotation on the engine pod instead of trying to override the canonical -# wiring out of existence. +# Admission HARD-REJECTS overrides that overlap the typed MP adapter's reserved +# args/env: `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager`, +# `PYTHONHASHSEED`, and `INFERENCECACHE_FAIL_OPEN`. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -40,26 +37,29 @@ spec: args: - "--max-model-len" - "8192" - # Override LMCACHE_CHUNK_SIZE — a documented tunable (NOT reserved), - # so the merge replaces the adapter's default of "256" with this - # value. INFERENCECACHE_FAIL_OPEN / VLLM_USE_V1 / LMCACHE_REMOTE_URL - # are reserved and would be rejected at admission if listed here. env: - - name: LMCACHE_CHUNK_SIZE - value: "512" - # Free-form engine env the user wants the engine container to see. - # The admission rule only fires for reserved Names; FOO is not - # reserved, so it's appended verbatim. + # Free-form engine env appended after the typed MP wire. - name: FOO value: bar engineSelector: matchLabels: app.kubernetes.io/name: vllm + lmCache: + topology: PodLocal + chunkSizeTokens: 512 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: # Served model identifier the matched engine pods are loaded with. modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - image: lmcache/standalone:v0.4.7 diff --git a/config/samples/cachebackend-external.yaml b/config/samples/cachebackend-external.yaml index 18b61065..0ae047e6 100644 --- a/config/samples/cachebackend-external.yaml +++ b/config/samples/cachebackend-external.yaml @@ -2,35 +2,15 @@ # # SPDX-License-Identifier: Apache-2.0 -# An External CacheBackend points the controller at a pre-existing remote -# cache the operator manages themselves. The controller does NOT provision -# pods for this backend (no Deployment, no Service, no HPA); it only mirrors -# spec.remoteStorage.endpoint into status.endpoint and marks the CR Ready as -# soon as admission accepts the spec. The pod-mutating Pod admission webhook then -# wires engine pods matching spec.engineSelector to that endpoint with the -# same LMCache engine wire format the managed-LMCache adapter uses (env vars -# LMCACHE_REMOTE_URL/SERDE/CHUNK_SIZE/LOCAL_CPU/MAX_LOCAL_CPU_SIZE + -# VLLM_USE_V1 + the LMCache `--kv-transfer-config` arg), so the engine cannot -# tell whether the cache it talks to was provisioned by the controller or by -# the operator out-of-band. +# Typed vLLM PodLocal LMCache MP with an externally managed Redis L3. The +# controller provisions no remote provider workload; the injected MP server +# sidecar connects to spec.remoteStorage.endpoint. The endpoint is RESP on a +# trusted private network: LMCache 0.5.3 authentication is supported, but TLS is +# not, and admission rejects a TLS block rather than silently ignoring it. # -# Use cases: -# * customers with a pre-existing LMCache deployment they don't want -# re-provisioned; -# * multi-cluster / cross-cluster cache sharing; -# * testing engine-side wiring against a fixture without spinning up a -# backend. -# -# Admission rules to be aware of: -# * remoteStorage.endpoint is REQUIRED for External ownership and rejected -# for Managed ownership; -# * an endpoint that resolves into an in-cluster Service in a -# different namespace than this CacheBackend requires the explicit -# opt-in spec.allowCrossNamespace=true (the rule does NOT fire for -# external hostnames / IPs); -# * the controller does not probe the endpoint for reachability — it -# trusts the operator. The CR goes Ready as soon as admission accepts -# spec.remoteStorage.endpoint. +# This sample explicitly chooses Redis. A legacy external LMCacheServer endpoint +# cannot be converted automatically because replacing lm:// changes sharing and +# data-plane semantics. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -40,24 +20,28 @@ metadata: spec: runtime: VLLM type: LMCache - # Address of the pre-existing cache server. Two accepted shapes: - # - bare `host:port` (canonical; the LMCache adapter adds the - # `lm://` scheme on engine injection) - # - `lm://host:port` (operators who prefer to be explicit) - # Whichever shape the operator writes, the reconciler mirrors the - # trimmed value into status.endpoint — so status carries the same - # form the operator typed (bare or lm://-prefixed), distinct from - # managed backends where status.endpoint is always engine-agnostic - # bare host:port from the rendered Service. The injection helper is - # lenient about an already-prefixed `lm://` value (the prefix is - # preserved rather than doubled). Admission requires a non-empty - # host AND port; IPv6 must be bracketed (`[::1]:8200`). integration: role: ReadWrite engineSelector: matchLabels: app.kubernetes.io/name: vllm + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi remoteStorage: - provider: LMCacheServer + provider: Redis ownership: External - endpoint: my-cache.example.com:8200 + endpoint: redis.example.internal:6379 diff --git a/config/samples/cachebackend-lmcache-cpu.yaml b/config/samples/cachebackend-lmcache-cpu.yaml deleted file mode 100644 index 64d0e0f3..00000000 --- a/config/samples/cachebackend-lmcache-cpu.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# Standalone lmcache-server backend for substrate validation on a GPU-free -# cluster (e.g. kind). The lmcache-server itself is always CPU-only (its -# storage device is `cpu` in-memory) regardless of what engine attaches to -# it — there is no separate CPU/GPU "profile" anymore: the cache server is -# engine-agnostic and the engine choice (CPU vs GPU image) lives on the -# user-owned vLLM Deployment, not on the CacheBackend. -# -# Apply this alongside a vLLM Deployment whose labels match -# `spec.engineSelector`; the controller's mutating Pod admission webhook -# auto-wires the engine pods to the resolved cache endpoint published in -# `status.endpoint` (an engine-agnostic host:port — the adapter adds the -# `lm://` scheme on the engine env). The webhook injects the engine-side -# env vars + `--kv-transfer-config` arg at pod admission (see -# docs/design/cachebackend-api.md §"Mutating Pod webhook (engine wiring)" -# for the full behavior, including the -# `inferencecache.io/skip-inject` per-pod opt-out). -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - labels: - app.kubernetes.io/name: inference-cache - name: cachebackend-lmcache-cpu -spec: - runtime: VLLM - type: LMCache - deploymentKind: Deployment - replicas: 1 - integration: - role: ReadWrite - engineSelector: - matchLabels: - app.kubernetes.io/name: vllm - observation: - # Served model identifier the matched engine pods are loaded with. - # Plumbed to the auto-attached kvevent-subscriber sidecar's --model-id - # so the index keys per-replica entries by model. Omit (or set empty) - # to skip the subscriber sidecar (engine wiring still happens). - modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - # Pin to a real digest for non-local runs. - image: lmcache/standalone:v0.4.7 diff --git a/config/samples/cachebackend-lmcache.yaml b/config/samples/cachebackend-lmcache.yaml index 8bd999cb..3126a93a 100644 --- a/config/samples/cachebackend-lmcache.yaml +++ b/config/samples/cachebackend-lmcache.yaml @@ -2,28 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -# A managed LMCache backend the controller reconciles into a standalone -# lmcache-server Deployment + Service. Engine pods (vLLM) are user-owned; -# spec.engineSelector identifies the pods the controller's mutating Pod -# admission webhook auto-wires to the resolved cache endpoint published -# in status.endpoint (as an engine-agnostic host:port — the adapter adds -# the lm:// scheme on the engine env). Pods labeled to match are -# auto-injected at admission with LMCACHE_REMOTE_URL/SERDE/CHUNK_SIZE/ -# LOCAL_CPU/MAX_LOCAL_CPU_SIZE + VLLM_USE_V1 + the LMCache -# `--kv-transfer-config` arg; user-set env/args on the pod template -# survive injection (the adapter merges, never clobbers). A pod can opt -# out per-instance with the annotation `inferencecache.io/skip-inject`. -# -# Phase 1 manages a Deployment + ClusterIP Service on port 65432 (the -# canonical LMCache lm:// port), plus an optional HorizontalPodAutoscaler -# from spec.autoscaling. The lmcache-server keeps KV in memory; for a -# durable / shared cache pick a backend designed for it (the planned -# Mooncake backend) rather than a volume knob on this one — see -# docs/design/lmcache-server-persistence.md. -# -# The engine cache and remote provider are explicit and independently owned: -# spec.lmCache configures the vLLM-side connector/host tier, while -# spec.remoteStorage.lmCacheServer configures the managed server workload. +# Canonical vLLM PodLocal LMCache MP backend with a controller-managed Redis +# remote L3. CacheBackend injects one native LMCache MP server sidecar per +# matching engine Pod and points it at Redis; it never replaces the +# inference-owner's engine image. This sample explicitly selects Redis to keep +# cross-Pod sharing. It is not an automatic mapping from the legacy +# LMCacheServer provider, whose lm:// data plane had different semantics. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -33,32 +17,38 @@ metadata: spec: runtime: VLLM type: LMCache - lmCache: - chunkSizeTokens: 256 - hostMemory: - capacity: 20Gi - deploymentKind: Deployment - replicas: 1 integration: role: ReadWrite engineSelector: matchLabels: app.kubernetes.io/name: vllm + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: - # Served model identifier the matched engine pods are loaded with. - # Plumbed to the auto-attached kvevent-subscriber sidecar's --model-id - # so the index keys per-replica entries by model. Omit (or set empty) - # to skip the subscriber sidecar (engine wiring still happens). modelID: meta-llama/Meta-Llama-3-8B-Instruct remoteStorage: - provider: LMCacheServer + provider: Redis ownership: Managed - lmCacheServer: - # Pin to an `@sha256:` digest for non-local runs and keep it - # wire-compatible with the engine's lmcache client. - image: lmcache/standalone:v0.4.7 + redis: + image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: requests: + cpu: 500m memory: 4Gi limits: + cpu: "2" memory: 8Gi diff --git a/config/samples/cachebackend-mooncake.yaml b/config/samples/cachebackend-mooncake.yaml deleted file mode 100644 index fa56ff67..00000000 --- a/config/samples/cachebackend-mooncake.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# A managed Mooncake backend the controller reconciles into a standalone -# mooncake_master Deployment + Service. Mooncake is the durable / shared cache -# path (vs. the in-memory lmcache-server in cachebackend-lmcache.yaml) — -# durability is a backend choice, not a volume knob; see -# docs/design/lmcache-server-persistence.md. -# -# HOST NETWORKING IS REQUIRED, on both sides. Mooncake is not a single-endpoint -# server like lm://: it is a peer-to-peer transfer-engine mesh. The master on -# :50051 returns only a directory pointer ("this block lives on node B"), and -# the engine then dials that node's real IP on a dynamically negotiated port. A -# ClusterIP Service forwards only its declared ports, and CNI overlay pod IPs -# are not reachable for the mesh — so the controller renders the master with -# hostNetwork behind a HEADLESS Service, and engine pods must opt in via -# spec.integration.engineHostNetwork below. Consequences: the namespace must -# permit hostNetwork (Pod Security "restricted" rejects it), the master reserves -# its ports on its node, and NetworkPolicy stops constraining those listeners -# (it selects pod IPs). See docs/design/cachebackend-api.md. -# -# How the engine is wired: Mooncake integrates with vLLM as an LMCache *remote -# backend*. Engine pods (vLLM) are user-owned; spec.engineSelector identifies -# the pods the controller's mutating Pod admission webhook auto-wires to the -# resolved cache endpoint published in status.endpoint (an engine-agnostic -# host:port — the master's RPC address). Matched pods are auto-injected at -# admission with the SAME LMCache connector the LMCache backend uses -# (LMCACHE_REMOTE_URL/SERDE/CHUNK_SIZE/LOCAL_CPU/MAX_LOCAL_CPU_SIZE + -# VLLM_USE_V1 + the `--kv-transfer-config` arg) — the only difference is the -# remote-store URL scheme: LMCACHE_REMOTE_URL=mooncakestore:// -# instead of lm://. User-set env/args on the pod template survive injection -# (the adapter merges, never clobbers). A pod can opt out per-instance with the -# annotation `inferencecache.io/skip-inject`. -# -# The controller manages a hostNetwork Deployment (Recreate strategy) + a -# HEADLESS Service exposing the Mooncake master RPC port (50051, the -# mooncakestore:// endpoint the engine dials) plus the master's embedded HTTP -# metadata port (8080). The Service DNS name published in status.endpoint -# therefore resolves straight to the master's node IP, with every port reachable. -# -# The master is a SINGLETON: spec.replicas > 1 and spec.autoscaling are rejected -# at admission, because a second master either cannot bind the first's node ports -# or comes up independently and silently splits the store. -# -# Operator note — transfer-engine config: the Mooncake transfer engine's static -# tuning (metadata_server, protocol tcp/rdma, device_name, segment sizes) lives -# in LMCache's extra_config, supplied to the engine via an operator-provided -# config file (LMCACHE_CONFIG_FILE / MOONCAKE_CONFIG_PATH) — it is not part of -# the auto-injected env. The defaults (P2P-handshake metadata) cover the -# simplest deployment; pin the rest for a real RDMA/HTTP-metadata setup. Keep -# the master image wire-compatible with the engine's mooncake-transfer-engine -# pip package version (see docs/design/cachebackend-api.md). -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - labels: - app.kubernetes.io/name: inference-cache - name: cachebackend-mooncake -spec: - runtime: VLLM - type: LMCache - deploymentKind: Deployment - replicas: 1 - integration: - role: ReadWrite - # REQUIRED for a working Mooncake data plane. Moves matched engine pods onto - # the host network so they can dial the mesh's real node IPs; an overlay pod - # IP cannot, and the backend would report Ready while transferring zero KV - # (admission warns when this is unset). - # - # Opt-in, never injected by default: hostNetwork is a privilege, and mutating - # webhooks run BEFORE Pod Security validation — so silently adding it would - # turn a working engine pod into one a "restricted" namespace rejects, with - # an error naming Pod Security rather than this controller. Setting it here - # is the operator's explicit acknowledgement. Rejected on backend types that - # do not need it, so it can never sit inert. - engineHostNetwork: true - engineSelector: - matchLabels: - app.kubernetes.io/name: vllm - observation: - # Served model identifier the matched engine pods are loaded with. - # Plumbed to the auto-attached kvevent-subscriber sidecar's --model-id - # so the index keys per-replica entries by model. Omit (or set empty) - # to skip the subscriber sidecar (engine wiring still happens). - modelID: meta-llama/Meta-Llama-3-8B-Instruct - remoteStorage: - provider: Mooncake - ownership: Managed - mooncake: - # Matches the pinned adapter default. Fully qualified (docker.io/...) so - # CRI-O nodes without short-name resolution configured don't reject it; - # version-aligned with mooncake-transfer-engine 0.3.11.post1 on PyPI. The - # master entrypoint + RPC/metadata/metrics ports are confirmed against the - # real image on a live cluster; digest-pinning is the remaining hardening. - # A wrong image surfaces as the CacheBackend staying Ready=False (the RPC - # readiness probe never passes), never a silent miss, and the cache is - # fail-open regardless. Pin to an `@sha256:` digest for non-local runs. - image: docker.io/kvcacheai/mooncake:0.3.11.post1 - # Override `command` to change the master's flags (a different - # metadata backend, HA mode, etc.). The default launches the master with - # its RPC port, Prometheus metrics port, and the embedded HTTP metadata - # server so the simplest deployment needs no external etcd/redis. Keep the - # RPC port (50051) and HTTP metadata port (8080) as-is: the Service, - # readiness probe, and status.endpoint are pinned to them, so changing the - # ports here would leave the engine wire pointing at dead ports. diff --git a/config/samples/cachebackend-sglang-host-only.yaml b/config/samples/cachebackend-sglang-host-only.yaml index d6fc062a..399c4620 100644 --- a/config/samples/cachebackend-sglang-host-only.yaml +++ b/config/samples/cachebackend-sglang-host-only.yaml @@ -2,10 +2,8 @@ # # SPDX-License-Identifier: Apache-2.0 -# SGLang with LMCache's node-local host-memory tier and no remote provider. -# Omitting spec.remoteStorage is intentional: the controller creates no -# Deployment or Service, and the runtime adapter injects an LMCache MP worker -# without an --l2-adapter. +# Typed SGLang PodLocal LMCache MP with no remote L3. L1 capacity is per engine +# Pod; omitting spec.remoteStorage intentionally provides no cross-Pod sharing. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -13,14 +11,26 @@ metadata: spec: runtime: SGLang type: LMCache - lmCache: - chunkSizeTokens: 256 - hostMemory: - capacity: 32Gi - observation: - modelID: meta-llama/Meta-Llama-3-8B-Instruct integration: role: ReadWrite engineSelector: matchLabels: app.kubernetes.io/name: sglang + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-sglang.yaml b/config/samples/cachebackend-sglang.yaml index 2d350ae3..770a9874 100644 --- a/config/samples/cachebackend-sglang.yaml +++ b/config/samples/cachebackend-sglang.yaml @@ -2,47 +2,11 @@ # # SPDX-License-Identifier: Apache-2.0 -# SGLang with an engine-side LMCache host tier and an explicitly selected -# Managed Redis remote tier. SGLang drives LMCache in MULTIPROCESS (MP) mode: -# - spec.runtime + spec.type select only the SGLang/LMCache engine wire; -# - spec.remoteStorage independently selects Managed Redis, published as -# status.endpoint; and -# - the mutating Pod webhook renders the MP engine wire on each matched pod: a -# node-local MP-worker native sidecar that writes an --lmcache-config-file -# (mp_host/mp_port) and offloads to that Redis, plus --enable-lmcache and -# LMCACHE_USE_EXPERIMENTAL=True on the engine container. -# GPU-validated end to end. Full design: docs/design/sglang-lmcache-mp-mode.md; -# key/field reference: docs/design/cachebackend-api.md "SGLang engine support". -# -# CLUSTER REQUIREMENT: Kubernetes >= 1.29 — the MP worker is a native sidecar (an -# initContainers entry with restartPolicy: Always), which older apiservers do not -# understand. vLLM+LMCache and the routing-only (EventsOnly) path have no such floor. -# -# spec.engineSelector identifies the user-owned SGLang pods the webhook auto-wires. -# SGLang turns LMCache on through a different launch surface than vLLM, so matched -# pods are injected with: -# - --enable-lmcache (SGLang's store_true flag; replaces vLLM's --kv-transfer-config) -# - --lmcache-config-file (the MP config the worker writes; MP mode aborts without it) -# - LMCACHE_USE_EXPERIMENTAL=True + INFERENCECACHE_FAIL_OPEN -# The old lm:// env (LMCACHE_REMOTE_URL and the LMCACHE_REMOTE_SERDE / CHUNK_SIZE / -# LOCAL_CPU / MAX_LOCAL_CPU_SIZE tunables) is NOT injected — MP mode ignores it. -# VLLM_USE_V1 and PYTHONHASHSEED are vLLM-only and likewise absent (SGLang has no v1 -# codepath; its sha256 prefix hashing does not depend on PYTHONHASHSEED). Unrelated -# user-set env/args on the pod template survive -# injection — the adapter merges, upserting only the keys it owns (a pod-template -# value colliding with an adapter-owned key is overwritten, not rejected). The -# hard-reject is a separate surface: a `spec.integration.engineOverrides` entry that -# overrides or suppresses a RESERVED arg/env is rejected at admission. A pod can opt -# out per-instance with the annotation `inferencecache.io/skip-inject`. -# -# The auto-attached kvevent-subscriber sidecar tags every report -# `--hash-scheme=sglang`, keeping SGLang prefixes in an index domain disjoint from -# vLLM's (no cross-engine false hits on identical prefix bytes). SGLang emits the -# same ZMQ BlockStored/BlockRemoved/AllBlocksCleared wire vLLM does, so the shipped -# subscriber decodes it unchanged — but the engine must be launched with -# `--kv-events-config '{"publisher":"zmq","endpoint":"tcp://*:5557","topic":"kv-events"}'` -# for the publisher to be active (the adapter wires the cache offload, not the -# event publisher — same division as the vLLM path). +# Typed SGLang PodLocal LMCache MP with a controller-managed Redis L3. The +# webhook injects the common lmcache-mp-server native sidecar plus SGLang's +# --enable-lmcache/--lmcache-config-file launch surface. Kubernetes 1.29 or +# newer is required for native sidecars. SGLang TP>1 is outside the validated +# migration baseline. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -52,35 +16,38 @@ metadata: spec: runtime: SGLang type: LMCache - deploymentKind: Deployment - replicas: 1 integration: - # LMCache currently admits only ReadWrite for every engine; directional - # roles require a connector that has been validated to enforce them. role: ReadWrite engineSelector: matchLabels: app.kubernetes.io/name: sglang lmCache: + topology: PodLocal chunkSizeTokens: 256 - hostMemory: - capacity: 4Gi + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: - # Served model identifier the matched SGLang pods are loaded with. Plumbed to - # the auto-attached kvevent-subscriber sidecar's --model-id so the index keys - # per-replica entries by model. Omit (or set empty) to skip the subscriber - # sidecar (engine wiring still happens). modelID: meta-llama/Meta-Llama-3-8B-Instruct remoteStorage: provider: Redis ownership: Managed redis: - # Image for the managed Redis L2 store the MP worker offloads to (the - # worker's `resp` --l2-adapter). Pin to an `@sha256:` digest for - # non-local runs. - image: docker.io/library/redis:7.4-alpine + image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: requests: + cpu: 500m memory: 4Gi limits: + cpu: "2" memory: 8Gi diff --git a/config/samples/cachebackend-with-engine.yaml b/config/samples/cachebackend-with-engine.yaml index 368fc52f..794ae24a 100644 --- a/config/samples/cachebackend-with-engine.yaml +++ b/config/samples/cachebackend-with-engine.yaml @@ -2,84 +2,13 @@ # # SPDX-License-Identifier: Apache-2.0 -# Paired sample: a CacheBackend and a matching engine Deployment. -# -# The label `app: qwen-demo` appears in TWO places — `CacheBackend.spec. -# engineSelector.matchLabels` and the Deployment's pod template -# `metadata.labels`. That label match is what binds the CR to the engine -# pods: the controller's mutating Pod admission webhook intercepts each -# pod CREATE, finds the matching CacheBackend, and injects the LMCache -# engine wiring (env vars + `--kv-transfer-config` arg). The -# kvevent-subscriber sidecar is appended in addition only when the -# controller is started with `--kvevent-subscriber-image` set (empty -# by default) AND the CR has a model id configured; otherwise the -# engine is wired without the sidecar. Drift the labels apart and the -# webhook silently no-ops, the engine runs uncached, and `kubectl get -# cachebackend` reports `Matched: 0`. -# See docs/concepts/cachebackend-engine-binding.md for the lifecycle and -# the failure-mode table. -# -# Resource naming: the CR is `qwen-demo-cache` and the engine Deployment -# is `qwen-engine`. They are deliberately named differently because the -# controller reconciles the CR into an `lmcache-server` Deployment whose -# name equals the CR's name — sharing the name with the engine Deployment -# would collide on Create. The label `app: qwen-demo` is what does the -# binding work, not the resource names. -# -# Apply order matters because the mutating Pod webhook fail-opens (admits -# the pod unmodified) when the matched CacheBackend's `status.endpoint` -# has not been published yet. Admission is CREATE-only, so a pod that -# loses this race stays uncached for its whole lifetime. -# -# Race-free recipe (recommended): -# -# # 1. Create only the CR (first YAML document; the `---` separator -# # marks where the engine Deployment begins). -# sed -n '/^---$/q;p' config/samples/cachebackend-with-engine.yaml \ -# | kubectl apply -f - -# -# # 2. Wait for the controller to publish status.endpoint. -# kubectl wait --for=jsonpath='{.status.endpoint}' \ -# cachebackend/qwen-demo-cache --timeout=60s -# -# # 3. Now apply the engine Deployment. -# sed -n '/^---$/,$p' config/samples/cachebackend-with-engine.yaml \ -# | tail -n +2 | kubectl apply -f - -# -# Quick demo (one-shot, may hit the race on a cold cluster): -# -# kubectl apply -f config/samples/cachebackend-with-engine.yaml -# -# Detecting the race after a one-shot apply: `status.matchedEnginePods` -# counts current label matches at reconcile, not whether the pod was -# injected at CREATE — a pod that raced past admission still has the -# right labels and still counts toward `MATCHED`, so the column alone -# can't tell you the race happened. The reliable race symptom is -# per-pod: the engine pod is missing the `inferencecache.io/injected-by` -# annotation and there is no `InjectedByCacheBackend` Event on it: -# -# kubectl get pod -l app=qwen-demo \ -# -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.annotations.inferencecache\.io/injected-by}{"\n"}{end}' -# -# A row with an empty second column raced. To fix, force admission to -# re-run by recreating the affected pods (admission is CREATE-only; -# editing labels on a live pod does NOT re-trigger): -# -# kubectl rollout restart deploy/qwen-engine -# -# Watch the binding land: -# -# kubectl get cachebackend qwen-demo-cache -w -# kubectl describe pod -l app=qwen-demo | grep -E 'Events|InjectedByCacheBackend' -# -# CPU-runnable: the engine image is vLLM's dedicated CPU build, so this -# pair works on kind without a GPU. Bump the host VM's memory to ~10 GiB -# before applying (vLLM CPU baseline is ~5 GiB + KV cache). -# -# Image tags are ARCH-SPECIFIC — vLLM publishes separate tags for x86_64 -# and arm64 and there is no shared "latest" multi-arch tag today. The -# default below targets x86_64 (the common cloud / CI shape). On Apple -# Silicon hosts switch the image to `vllm/vllm-openai-cpu:latest-arm64`. +# Paired typed-MP sample: one CacheBackend and one matching vLLM Deployment. +# The label `app: qwen-demo` is the binding; at Pod CREATE the webhook injects +# an LMCache MP native sidecar and the vLLM LMCacheMPConnector JSON. This example +# intentionally omits remoteStorage, so L1 is per Pod and no cross-Pod sharing +# is claimed. The engine image remains inference-owner supplied: normal engine +# startup is the authoritative check that it contains a compatible LMCache +# client. Kubernetes 1.29 or newer is required for the native sidecar. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -87,8 +16,6 @@ metadata: spec: runtime: VLLM type: LMCache - deploymentKind: Deployment - replicas: 1 integration: role: ReadWrite engineSelector: @@ -99,12 +26,22 @@ spec: # auto-attached kvevent-subscriber sidecar so the index keys # per-replica entries by model. modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - # Pin to a real digest for non-local runs. - image: lmcache/standalone:v0.4.7 + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi --- apiVersion: apps/v1 kind: Deployment @@ -158,4 +95,4 @@ spec: - name: shm emptyDir: medium: Memory - sizeLimit: 4Gi + sizeLimit: 5Gi diff --git a/config/samples/cachebackend-with-override.yaml b/config/samples/cachebackend-with-override.yaml index 6500be49..e055583c 100644 --- a/config/samples/cachebackend-with-override.yaml +++ b/config/samples/cachebackend-with-override.yaml @@ -2,12 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -# Paired CacheBackend + matching engine Deployment that exercises a small -# `spec.integration.engineOverrides` block. Shows how to tune chunk size -# and add a debug env WITHOUT losing the rest of the LMCache integration -# the controller's mutating Pod admission webhook injects. +# Paired typed-MP CacheBackend + matching engine Deployment that exercises a +# small `spec.integration.engineOverrides` block. LMCache chunk size is a typed +# `spec.lmCache.chunkSizeTokens` field; engineOverrides is used only for an +# extra engine environment value. # -# The override block is intentionally small (two env entries) — the goal +# The override block is intentionally small — the goal # is to demonstrate the cause -> effect of `engineOverrides`, not to # exercise all four primitives in one sample. See # docs/concepts/cachebackend-engine-overrides.md for the full menu (env @@ -18,24 +18,10 @@ # # kubectl apply -f config/samples/cachebackend-with-override.yaml # -# The pod-mutating webhook fail-opens when CacheBackend.status.endpoint is -# still empty, so a fresh-cluster apply that races the CacheBackend -# reconciler can create the first engine pod BEFORE the LMCache server -# endpoint is published, leaving that first pod unwired. If the engine -# container env is missing the canonical LMCache_* / VLLM_USE_V1 entries -# after a first apply, wait for `kubectl get cachebackend qwen-demo -o -# jsonpath='{.status.endpoint}'` to be non-empty and then -# `kubectl rollout restart deployment/qwen-demo-engine` so the next pod -# admits with status.endpoint populated. -# -# Once admitted with a non-empty status.endpoint, -# `kubectl describe pod -l app=qwen-demo` should show the engine -# container carrying: -# LMCACHE_CHUNK_SIZE = "64" (override replaced the canonical "256") -# LMCACHE_LOG_LEVEL = "DEBUG" (appended; adapter doesn't inject this) -# plus the rest of the canonical LMCache wiring (LMCACHE_REMOTE_URL, -# --kv-transfer-config, VLLM_USE_V1, etc.) — see the concept doc's -# "Baseline" section for the full canonical set. +# After admission the engine carries `LMCACHE_LOG_LEVEL=DEBUG` in addition to +# the reserved MP connector JSON, deterministic hash seed, and hybrid-cache +# manager guard. The MP server sidecar receives chunk size 64 from the typed +# CacheBackend field. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -50,18 +36,14 @@ spec: integration: role: ReadWrite # Amend the engine container the mutating Pod webhook injects. - # Reserved entries (--kv-transfer-config, LMCACHE_REMOTE_URL, - # VLLM_USE_V1, INFERENCECACHE_FAIL_OPEN) are HARD-REJECTED at + # Reserved entries (--kv-transfer-config, + # --disable-hybrid-kv-cache-manager, PYTHONHASHSEED, and + # INFERENCECACHE_FAIL_OPEN) are HARD-REJECTED at # admission with a field-scoped error; non-reserved entries pass # through and merge. engineOverrides: env: - # Upsert: name matches an adapter-injected env, so the canonical - # value ("256") is replaced with "64". Drop chunk size for nodes - # where the default eats too much per-replica memory. - - name: LMCACHE_CHUNK_SIZE - value: "64" - # Append: name not in the adapter's canonical set, so it's added + # Append: name is not in the adapter's canonical set, so it is added # to the engine env unchanged. Useful for triaging cache behavior # against the LMCache log stream. - name: LMCACHE_LOG_LEVEL @@ -74,17 +56,26 @@ spec: # Plumbed to the auto-attached kvevent-subscriber sidecar's --model-id # so the index keys per-replica entries by model. modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - # Pin to a real digest for non-local runs. - image: lmcache/standalone:v0.4.7 + lmCache: + topology: PodLocal + chunkSizeTokens: 64 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi --- # Engine Deployment. Apply this alongside the CacheBackend above; the -# order matters (see the top-of-file comment). Pod injection is -# CREATE-only, so a pod admitted before the CacheBackend's -# status.endpoint is published will run unwired until it is recreated. +# Pod injection is CREATE-only; applying the CacheBackend before the Deployment +# remains the clearest operational order. # The mutating Pod admission webhook intercepts pods matching the # CacheBackend's `engineSelector` and injects the LMCache wiring + # (above) the engineOverrides amendments onto the `vllm` container at @@ -92,12 +83,8 @@ spec: # # The Deployment name is `qwen-demo-engine`, distinct from the CacheBackend # name (`qwen-demo`), because the CacheBackend reconciler stands up its own -# Deployment named after the CacheBackend for the managed lmcache-server. -# Reusing the same name would either collide on apply or get reconciled -# back to the lmcache-server shape. Binding to the CacheBackend goes -# through the `app: qwen-demo` label match in `spec.engineSelector` above, -# not through the Deployment name, so the engine Deployment can be named -# anything that does not collide with the managed pod. +# Binding to the CacheBackend goes through the `app: qwen-demo` label match in +# `spec.engineSelector`, not through the Deployment name. # # CPU vLLM image and flags mirror docs/reference-stack/manifests/cpu-local/ # deployment.yaml (the standalone CPU manifest that has been verified @@ -166,4 +153,4 @@ spec: - name: shm emptyDir: medium: Memory - sizeLimit: 4Gi + sizeLimit: 5Gi diff --git a/config/samples/recipe-cpu-dev.yaml b/config/samples/recipe-cpu-dev.yaml index a5aea792..18da47e6 100644 --- a/config/samples/recipe-cpu-dev.yaml +++ b/config/samples/recipe-cpu-dev.yaml @@ -4,9 +4,10 @@ # Recipe: CPU dev — fastest path to a working cache on a laptop / kind cluster. # -# Scenario: stand up a cache-aware backend with no GPU. This pairs a managed -# LMCache backend with a tiny vLLM engine running on vLLM's dedicated CPU build. -# A single `kubectl apply -f` wires the engine to the cache (KV offload/reuse) +# Scenario: exercise the typed PodLocal binding shape without requiring a GPU +# for admission. This pairs a host-only LMCache MP CacheBackend with a tiny vLLM +# CPU Deployment. A single `kubectl apply -f` asks the webhook to wire the engine +# to the cache # and lets the backend produce LookupRoute hints. Acting on those hints to # actually route requests is the gateway's job — that integration ships as a # separate (deferred) cache-aware-routing recipe, not this one. @@ -31,27 +32,19 @@ # the Troubleshooting section of docs/quickstart.md for the per-reason # runbook. # -# CPU caveat: LMCache offload is functional on the CPU image (the repo's -# CPU substrate canary exercises cross-pod KV reuse via the lmcache-server), but -# representative offload performance wants a GPU — see docs/reference-stack/. +# Compatibility caveat: the inference owner must supply an engine image that +# contains the selected LMCache MP client/API. This repository validates the +# manifest and webhook mutation without a GPU; it does not claim that every +# vLLM CPU image bundles the connector. Normal engine startup is authoritative. # # What binds the two objects: the label `app: cpu-dev` appears in BOTH # `CacheBackend.spec.engineSelector.matchLabels` AND the engine Deployment's # pod-template labels. The mutating Pod webhook intercepts each engine pod at # CREATE, finds the matching CacheBackend, and injects the LMCache engine -# wiring (env + `--kv-transfer-config`). Drift the labels apart and the +# wiring (native MP sidecar + `--kv-transfer-config`). Drift the labels apart and the # webhook silently no-ops — `kubectl get cachebackend` then shows `MATCHED: 0` # and the engine runs uncached. See docs/concepts/cachebackend-engine-binding.md. # -# Apply order note: the pod webhook fail-opens (admits the pod unmodified) -# while `CacheBackend.status.endpoint` is still empty, so a fresh-cluster -# one-shot apply can race — the first engine pod may come up unwired. If so, -# wait for the endpoint and recreate the engine pods: -# -# kubectl wait --for=jsonpath='{.status.endpoint}' \ -# cachebackend/cpu-dev-cache --timeout=60s -# kubectl rollout restart deploy/cpu-dev-engine -# # CPU sizing: vLLM's CPU runtime baseline is ~5 GiB before any KV cache. Give # the kind node / VM ~10 GiB. Image tags are arch-specific (no shared multi-arch # `latest`): the default below is x86_64; on Apple Silicon swap both the image @@ -69,17 +62,27 @@ spec: engineSelector: matchLabels: app: cpu-dev # <-- binding label (1 of 2) + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 2Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 3Gi + limits: + cpu: "2" + memory: 4Gi observation: # Served model the engine pods load. Plumbed to the kvevent-subscriber # sidecar so the index keys per-replica entries by model. Tiny + ungated # so it runs on CPU with no auth token. modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - # Managed lmcache-server image. Pin to a digest for non-local runs. - image: lmcache/standalone:v0.4.7 --- apiVersion: apps/v1 kind: Deployment diff --git a/config/samples/recipe-external-cache.yaml b/config/samples/recipe-external-cache.yaml index 42527edb..7b370901 100644 --- a/config/samples/recipe-external-cache.yaml +++ b/config/samples/recipe-external-cache.yaml @@ -2,22 +2,16 @@ # # SPDX-License-Identifier: Apache-2.0 -# Recipe: External cache — point the operator at a cache you manage yourself. +# Recipe: External Redis L3 — point the MP server at Redis you manage. # -# Scenario: you already run an lmcache-server (or share one across clusters) -# and don't want the controller to provision a new one. External ownership -# makes the controller skip all provisioning (no Deployment, no Service, no -# HPA); it mirrors `spec.remoteStorage.endpoint` into `status.endpoint` and -# marks the CR Ready as soon as admission accepts the spec. The pod webhook -# then wires -# engine pods matching `spec.engineSelector` to that endpoint with the same -# LMCache wire format a managed backend uses — the engine can't tell the -# difference. +# Scenario: you already run Redis and do not want the controller to provision +# it. External ownership skips the remote-provider Deployment/Service/HPA. The +# pod webhook still injects one PodLocal LMCache MP server per engine Pod, and +# that server connects to the explicit RESP endpoint. # # Admission rules for External ownership: # * remoteStorage.endpoint is REQUIRED for External ownership and REJECTED -# for Managed ownership. Accepted shapes are bare `host:port` or -# `lm://host:port`. +# for Managed ownership. Redis accepts bare `host:port`. # A non-empty port is required; IPv6 literals must be bracketed ([::1]:8200). # * If the endpoint resolves to an in-cluster Service in a DIFFERENT namespace # than this CR, set `spec.allowCrossNamespace: true` to acknowledge the @@ -25,7 +19,7 @@ # * The controller does NOT probe the endpoint — it trusts the operator. # # This recipe ships only the CacheBackend + a matching engine Deployment; the -# cache server itself is assumed to already exist at the endpoint below. +# Redis itself is assumed to already exist at the endpoint below. # # OBSERVABILITY PREREQUISITE: per-replica index entries are reported only when # the kvevent-subscriber sidecar is auto-attached, which (as for managed @@ -51,16 +45,31 @@ spec: engineSelector: matchLabels: app: external-demo + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 2Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 3Gi + limits: + cpu: "2" + memory: 4Gi observation: # Model the engine pods load — keys per-replica index entries when the # subscriber sidecar is attached (see the prerequisite note above). modelID: Qwen/Qwen2.5-0.5B-Instruct remoteStorage: - provider: LMCacheServer + provider: Redis ownership: External - # Address of the pre-existing cache server you manage. Swap for your own. - # Bare host:port shown; `lm://host:port` is also accepted. - endpoint: my-cache.example.com:8200 + # Address of the pre-existing Redis service you manage. Swap for your own. + endpoint: redis.example.internal:6379 --- apiVersion: apps/v1 kind: Deployment diff --git a/config/samples/recipe-gpu-production.yaml b/config/samples/recipe-gpu-production.yaml index 59285a1e..c9689b07 100644 --- a/config/samples/recipe-gpu-production.yaml +++ b/config/samples/recipe-gpu-production.yaml @@ -7,30 +7,27 @@ # images (digest-pin them) and tune the model / resources / quotas for your # cluster before applying. # -# Scenario: a real model on GPU engine pods, a managed LMCache backend that -# autoscales under load, and a per-namespace CachePolicy with production-grade -# TTLs. This file carries three objects: the CacheBackend (with an autoscaling -# block), a CachePolicy for the namespace, and the GPU engine Deployment. +# Scenario: a real model on GPU engine pods, typed PodLocal LMCache MP with an +# explicitly selected managed Redis L3 for cross-Pod sharing, and a +# per-namespace CachePolicy with production-grade TTLs. # # Differences from the CPU dev recipe: # * The engine Deployment requests a GPU (`nvidia.com/gpu: 1`) and uses the # CUDA vLLM image — schedule it onto GPU nodes. -# * `spec.autoscaling` is set, so the controller reconciles a -# HorizontalPodAutoscaler that drives the MANAGED lmcache-server workload's -# replica count (it overrides spec.replicas while active). This autoscales -# the cache server, not the engine — scale the engine via its own HPA. +# * Redis is intentionally a singleton soft-state L3; this recipe does not +# map the legacy LMCacheServer autoscaling shape because independent Redis +# replicas would partition the keyspace rather than preserve semantics. # * A CachePolicy tunes eviction for the namespace (see the CachePolicy doc # pointer in config/samples/README.md). # # Binding works exactly as in recipe-cpu-dev.yaml: the `app: prod-llm` label on # the engine pod template matches `spec.engineSelector.matchLabels`. Same -# fail-open-on-empty-endpoint apply race applies — wait for status.endpoint and -# `kubectl rollout restart deploy/prod-llm-engine` if the first pod comes up -# unwired. See docs/concepts/cachebackend-engine-binding.md. +# binding remains CREATE-time; apply the CacheBackend before the engine +# Deployment. See docs/concepts/cachebackend-engine-binding.md. # # IMAGES ARE PLACEHOLDERS — replace before production. The engine image below # uses a moving `:latest` tag so the recipe stays self-contained; the managed -# cache-server image is pinned to a version (not `:latest`). Pin each to +# MP server and Redis images are pinned. Pin the engine to # an immutable digest for a real deployment. # # READY/OBSERVABILITY PREREQUISITE: the backend reaches Ready=True and reports @@ -59,48 +56,41 @@ metadata: spec: runtime: VLLM type: LMCache - # Baseline replica count for the managed lmcache-server. The autoscaling - # block below takes over once the HPA is reconciled. - replicas: 2 - autoscaling: - minReplicas: 2 - maxReplicas: 6 - targetCPUUtilizationPercent: 70 - # Per-replica resource budget for the lmcache-server container. Omit - # remoteStorage.lmCacheServer.resources and the provider fallback supplies - # `requests.memory=4Gi`, `limits.memory=8Gi` — adequate for a small - # workload, conservative enough that the kubelet does not OOM-kill the - # cache pod from node-level memory pressure. The cgroup memory limit - # is still a hard ceiling: a working set larger than the configured - # limit will trigger an in-cgroup OOM-kill, so size the limit to your - # KV footprint. Production workloads should override the request/limit - # explicitly so the cache pod's cgroup matches what it will actually - # hold (and so the cluster-autoscaler reasons about correct headroom). - # 16Gi/24Gi below sizes for a Qwen2.5-7B prod shape — tune to yours. - # - # NOTE: this override is memory-only. The autoscaling block above - # requests a CPU-utilization HPA, so the controller fills in a 250m - # CPU request fallback automatically (an HPA needs a CPU request as - # its utilization denominator). Add `cpu: …` under requests below to - # pin the denominator explicitly if you want a different baseline. integration: role: ReadWrite engineSelector: matchLabels: app: prod-llm + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 16Gi + maxWorkers: 4 + resources: + requests: + cpu: "2" + memory: 17Gi + limits: + cpu: "4" + memory: 20Gi observation: # Real, ungated model (Apache-2.0) — no auth token required. modelID: Qwen/Qwen2.5-7B-Instruct remoteStorage: - provider: LMCacheServer + provider: Redis ownership: Managed - lmCacheServer: - # PLACEHOLDER — replace with a digest-pinned reference before production. - image: lmcache/standalone:v0.4.7 + redis: + image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: requests: + cpu: "2" memory: "16Gi" limits: + cpu: "4" memory: "24Gi" --- apiVersion: inferencecache.io/v1alpha1 @@ -195,4 +185,4 @@ spec: - name: shm emptyDir: medium: Memory - sizeLimit: 8Gi + sizeLimit: 17Gi diff --git a/config/samples/recipe-multi-tenant.yaml b/config/samples/recipe-multi-tenant.yaml index 030b2fb9..550aa74c 100644 --- a/config/samples/recipe-multi-tenant.yaml +++ b/config/samples/recipe-multi-tenant.yaml @@ -46,7 +46,8 @@ # sidecar is auto-attached, which requires the controller to run with # --kvevent-subscriber-image set (empty by default). Without it the engines are # wired to their caches but report no KV events, so status.indexEntries stays -# unset. CPU shape — runs without a GPU. +# unset. The manifests are admission-valid without a GPU; the inference owner +# must still supply an engine image containing the selected LMCache MP client. apiVersion: v1 kind: Namespace metadata: @@ -94,13 +95,24 @@ spec: engineSelector: matchLabels: app: tenant-engine + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 2Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 3Gi + limits: + cpu: "2" + memory: 4Gi observation: modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - image: lmcache/standalone:v0.4.7 --- apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend @@ -116,13 +128,24 @@ spec: engineSelector: matchLabels: app: tenant-engine + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 2Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 3Gi + limits: + cpu: "2" + memory: 4Gi observation: modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - image: lmcache/standalone:v0.4.7 --- apiVersion: apps/v1 kind: Deployment diff --git a/config/samples/recipe-tuning.yaml b/config/samples/recipe-tuning.yaml index bb548da6..be632cf4 100644 --- a/config/samples/recipe-tuning.yaml +++ b/config/samples/recipe-tuning.yaml @@ -4,20 +4,17 @@ # Recipe: Tuning — CPU dev shape plus a meaningful engineOverrides block. # -# Scenario: the canonical LMCache injection works, but you want to tune an -# adapter-injected knob (chunk size) and turn up LMCache logging for triage — -# WITHOUT losing the rest of the integration the pod webhook injects. +# Scenario: tune the MP server's typed chunk size and add an engine-side logging +# environment value without losing the rest of the injected integration. # # `spec.integration.engineOverrides` amends the engine container on top of the -# adapter's canonical injection. Two primitives are shown here: -# * UPSERT — `LMCACHE_CHUNK_SIZE` matches an adapter-owned env, so the override -# value (128) REPLACES the canonical default. Drop chunk size on nodes where -# the default eats too much per-replica memory. -# * APPEND — `LMCACHE_LOG_LEVEL` is not in the adapter's canonical set, so it +# adapter's canonical injection. `LMCACHE_LOG_LEVEL` is not in the adapter's +# canonical set, so it # is added unchanged. Useful for reading the LMCache log stream during triage. # # IMPORTANT: overrides that overlap the adapter's RESERVED args/env -# (--kv-transfer-config, LMCACHE_REMOTE_URL, VLLM_USE_V1, INFERENCECACHE_FAIL_OPEN) +# (--kv-transfer-config, --disable-hybrid-kv-cache-manager, PYTHONHASHSEED, +# INFERENCECACHE_FAIL_OPEN) # are HARD-REJECTED at admission with a field-scoped error — they are required # for the integration to function. engineOverrides tunes; it does not disable. # To skip injection entirely on a pod, use the `inferencecache.io/skip-inject` @@ -44,22 +41,30 @@ spec: role: ReadWrite engineOverrides: env: - # Upsert: replaces the adapter's canonical chunk-size value. - - name: LMCACHE_CHUNK_SIZE - value: "128" # Append: adapter doesn't inject this, so it's added unchanged. - name: LMCACHE_LOG_LEVEL value: DEBUG engineSelector: matchLabels: app: tuning-demo + lmCache: + topology: PodLocal + chunkSizeTokens: 128 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 2Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 3Gi + limits: + cpu: "2" + memory: 4Gi observation: modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - image: lmcache/standalone:v0.4.7 --- apiVersion: apps/v1 kind: Deployment diff --git a/docs/concepts/cachebackend-engine-binding.md b/docs/concepts/cachebackend-engine-binding.md index 56dbc3d1..ba2b49d5 100644 --- a/docs/concepts/cachebackend-engine-binding.md +++ b/docs/concepts/cachebackend-engine-binding.md @@ -1,120 +1,115 @@ -# CacheBackend ↔ engine-pod binding +# CacheBackend ↔ engine-Pod binding -CacheBackend uses Kubernetes label selectors to find the engine pods it injects cache wiring into. This page explains the model, the lifecycle, and the common ways it goes wrong. +CacheBackend uses a namespaced label selector to find the inference-engine Pods +whose cache integration it should inject. The inference system owns the engine +Deployment and image; CacheBackend owns only the cache components and +engine-specific connector wire it adds. -## How it works +## Current LMCache flow -Three actors participate in the binding: +For `spec.type: LMCache`, current manifests declare +`spec.lmCache.topology: PodLocal`. At Pod CREATE, the webhook: -- **CacheBackend** — the namespaced CR you create. Its `spec.engineSelector.matchLabels` is a label selector over pods in the same namespace, with the same semantics as `Service.spec.selector`. -- **Engine pod** — a vLLM (or other supported runtime) pod, typically owned by a user-managed Deployment. Its `template.metadata.labels` are what the selector matches against. -- **Mutating Pod webhook** — the controller's admission webhook intercepts pod CREATE and stamps the matched engine pod with the LMCache engine wiring (env vars + CLI args). The kvevent-subscriber observation sidecar is appended in addition only when the controller is started with `--kvevent-subscriber-image` set; lifecycle step 3 below has the full conditions. +1. finds matching CacheBackends in the Pod's namespace; +2. selects the runtime-specific MP adapter; +3. injects one `lmcache-mp-server` native sidecar, shared `/dev/shm`, and the + vLLM or SGLang connector launch surface; +4. optionally binds the MP server to a Redis L3; and +5. stamps `inferencecache.io/injected-by` and + `inferencecache.io/injected-by-uid`. + +The engine image is never replaced or inspected. Normal engine initialization +is the authoritative compatibility check for the required connector/package. +PodLocal native sidecars require Kubernetes 1.29 or newer. ```text - +-----------------------+ - | CacheBackend (CR) | - | spec.engineSelector | - +-----------+-----------+ - | label-selector match (at pod CREATE) - v -+-----------------+ pod CREATE +------------+-----------+ -| Engine | -----------+----> | Mutating Pod webhook | -| Deployment | | | (matches selector; | -| template: | | | injects engine config;| -| labels: {...} | | | +subscriber sidecar*) | -+-----------------+ | +------------+-----------+ - | | - | v - | +----------+--------------+ - | | Engine pod | - | | env: LMCACHE_* | - | | args: --kv-... | - | | sidecar: subscriber* | - | +----------+--------------+ - | | - | v subscriber publishes - | +----------+-----------+ - | | lmcache-server pod | - +------> | (managed by the CR; | - | endpoint published | - | in status.endpoint) | - +----------------------+ +CacheBackend selector ──matches at Pod CREATE──▶ mutating webhook + │ + ▼ +engine Pod: engine + LMCache MP server sidecar + optional subscriber + │ + └── optional RESP ──▶ Redis L3 ``` -The match is evaluated **once at pod CREATE** by the mutating webhook. The wiring is sticky to the life of the pod; relabeling an existing pod does not re-evaluate it. To opt a pod out regardless of label match, set `inferencecache.io/skip-inject: "true"` on the pod template. Skipped pods are stamped with `inferencecache.io/inject-skipped: "skip-inject-annotation"` and receive a `SkippedByOperator` Event, so an intentional opt-out is distinguishable from selector drift. - -`*` The kvevent-subscriber sidecar is opt-in. It is appended only when the controller is started with `--kvevent-subscriber-image` set (empty by default) AND the matched CacheBackend has `spec.observation.modelID` configured; otherwise the engine is wired without it. The default install does not auto-attach the sidecar. - -> **Native SGLang HiCache exception.** `type: SGLangHiCache` is engine-local: -> the controller creates no backend workload or endpoint, and the webhook does -> not wait for `status.endpoint`. It injects the typed `spec.hiCache` launch -> flags directly into matching SGLang Pods. The LMCache lifecycle below remains -> the endpoint-bearing path. +Host-only PodLocal objects publish no endpoint. With external Redis, the +webhook uses `spec.remoteStorage.endpoint`; with managed Redis, it uses the +controller-resolved endpoint. Connector readiness and remote-storage readiness +are reported independently. ## Lifecycle -1. **Apply the CacheBackend.** `kubectl apply -f cachebackend.yaml`. The reconciler creates the managed lmcache-server Deployment + Service and publishes the resolved address in `status.endpoint`. -2. **Deploy the engine.** Apply an engine Deployment whose pod template labels include every key/value in `spec.engineSelector.matchLabels`. New pods from that Deployment hit the mutating webhook at admission time. -3. **The webhook claims matching pods (precondition: status.endpoint published).** When the matched CacheBackend has `status.endpoint` populated by the time admission runs, the webhook injects LMCache env vars, the `--kv-transfer-config` CLI arg, and stamps TWO annotations: `inferencecache.io/injected-by: /` (operator-readable identity, visible in `kubectl describe pod`) and `inferencecache.io/injected-by-uid: ` (the matched CR's `metadata.uid` — an apiserver-assigned identifier that the events controller cross-checks against the live CR's UID before emitting. The check catches casual copy-paste of an injected pod's annotations into a fresh template, but it isn't a security boundary: UIDs are readable metadata, so a pod creator with `get` RBAC on CacheBackends can stamp the pair correctly). When the controller is started with `--kvevent-subscriber-image` set (empty by default) AND the matched CacheBackend has a model id configured, the kvevent-subscriber sidecar is also appended; otherwise the engine is wired without the sidecar. A controller watching pods then validates the UID annotation against the live CR's UID and records a `Normal InjectedByCacheBackend` event on the now-persisted pod, visible in `kubectl describe pod`. (The event is emitted from a controller rather than the webhook because the apiserver does not assign `metadata.uid` until after mutating admission — an event recorded from the webhook would have `involvedObject.uid=""` and would not surface under describe. The UID annotation is what lets the controller distinguish a real webhook injection from a user-supplied `injected-by` annotation when the webhook is unreachable under `failurePolicy=Ignore`.) **If the CacheBackend's `status.endpoint` is empty at admission time** (the operator applied the engine Deployment before the reconciler had a chance to publish it), the webhook fail-opens — the pod is admitted unwired, no annotations are stamped, no Event is emitted. Because admission is CREATE-only, this is permanent for that pod; recovery is `kubectl rollout restart deploy/` so new pods re-enter admission. Note that `status.matchedEnginePods` still counts the unwired pod (the selector matches its labels), so `Matched > 0` does **not** guarantee the pod was injected — the per-pod annotations and Event are the authoritative wiring signals. -4. **KV events flow (when the sidecar is configured).** When the subscriber sidecar is present, it streams the engine's KV-cache events to the policy server's index, which surfaces them in `CacheBackend.status` (the index-participation status fields). Without the sidecar, the binding is still observable — `status.matchedEnginePods` and the per-pod annotation/Event still materialize from step 3 — but no KV events flow and the index-participation fields stay unset. -5. **Observe and debug.** `kubectl get cachebackend` shows the `Matched` column — the snapshot count of pods whose labels currently match. `kubectl describe pod ` shows which CacheBackend (if any) claimed it. +1. Apply the CacheBackend before creating engine Pods. +2. Create an engine Deployment whose Pod-template labels include every + `spec.engineSelector.matchLabels` entry. +3. Admission injects the complete MP wire atomically. A collision or invalid + Pod shape fails open without a partial mutation; inspect Pod annotations, + Events, and engine startup logs. +4. If `--kvevent-subscriber-image` is configured and + `spec.observation.modelID` is set, the webhook also adds the observation + sidecar. The subscriber reports metadata-only KV events to the policy index. +5. Recreate Pods after changing a CacheBackend. Injection is evaluated only at + Pod CREATE; relabeling or editing a running Pod does not re-run admission. ## Annotated example -A single CacheBackend with a matching engine Deployment. The label `app: qwen-demo` appears in two places — that's what binds them: - ```yaml apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: - name: qwen-demo-cache # <-- CR name; deliberately distinct from the engine Deployment name (see note below) + name: qwen-demo-cache spec: runtime: VLLM type: LMCache - integration: - role: ReadWrite engineSelector: matchLabels: - app: qwen-demo # <-- selector key/value (1 of 2; binding is by label, not by resource name) - observation: - modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: {} + app: qwen-demo + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi --- apiVersion: apps/v1 kind: Deployment metadata: - name: qwen-engine # <-- engine Deployment name; must differ from the CR name above (the controller reconciles the CR into an lmcache-server Deployment whose name equals the CR's name, so sharing names would collide on Create) + name: qwen-engine spec: - replicas: 1 selector: matchLabels: app: qwen-demo template: metadata: labels: - app: qwen-demo # <-- selector key/value (2 of 2; this is what the webhook sees) + app: qwen-demo spec: containers: - name: vllm - # Arch-tagged: swap to `:latest-arm64` on Apple Silicon hosts. - image: vllm/vllm-openai-cpu:latest-x86_64 - args: ["--model", "Qwen/Qwen2.5-0.5B-Instruct"] + image: example.invalid/runtime-owned-vllm-lmcache@sha256:0000000000000000000000000000000000000000000000000000000000000000 ``` -A copy-pasteable version of this pair ships at [`config/samples/cachebackend-with-engine.yaml`](../../config/samples/cachebackend-with-engine.yaml). +A fuller paired sample is +[`config/samples/cachebackend-with-engine.yaml`](../../config/samples/cachebackend-with-engine.yaml). ## Common failure modes -| Symptom | Cause | How to detect | Fix | -|---|---|---|---| -| Engine pod runs uncached; no LMCache env on its container | Selector and pod labels don't overlap (typo, drift after a Deployment rename, etc.) | `kubectl describe pod ` shows no `InjectedByCacheBackend` or `SkippedByOperator` event and neither `inferencecache.io/injected-by` nor `inferencecache.io/inject-skipped`; `kubectl get cachebackend` shows `Matched: 0`; `kubectl get cachebackend -o jsonpath='{.status.engineSelectorMessage}'` echoes the selector that matched no pods; `kubectl describe cachebackend ` shows a Normal `EngineSelectorUnmatched` Event when the CR first observes zero matches or transitions from matched to zero | Reconcile the label sets: either fix `engineSelector.matchLabels` on the CR or fix the Deployment's pod template labels | -| Multiple CacheBackends overlap on the same pod | Two CacheBackends in the namespace have selectors that both match | The webhook picks the lexicographically-first match by `metadata.name` (sort is deterministic so the picked CR is reproducible across re-admissions) and stamps `inferencecache.io/injected-by` on the pod — `kubectl describe pod` and that annotation name the CR that actually injected. There is no admission validator for selector overlap today; multi-match is a misconfiguration the operator must avoid by hand | Pick one CR for the pod; delete or narrow the other so each engine pod's labels match exactly one CacheBackend | -| Engine pod was labeled after creation but still uncached | Label match is evaluated once at pod CREATE; relabeling later has no effect | `kubectl describe pod ` shows no `InjectedByCacheBackend` event | Delete the pod (`kubectl delete pod `); the Deployment will recreate it and the new pod will re-enter admission | -| CacheBackend was deleted, but engine pods are still running with the old wiring | Wiring is sticky to the pod's lifetime; deleting the CR does not retract env vars from already-admitted pods | Engine logs show LMCache connect failures to a no-longer-existing Service | Rolling-restart the engine Deployment to admit fresh pods (which will match no CR and run uncached) | -| Engine pod intentionally runs uncached | The pod template has `inferencecache.io/skip-inject` set to a truthy value | `kubectl get pod -o jsonpath='{.metadata.annotations.inferencecache\.io/inject-skipped}'` returns `skip-inject-annotation`, and `kubectl describe pod ` shows a Normal `SkippedByOperator` Event | No fix needed if the opt-out was intentional; remove the skip annotation from the pod template and restart if the pod should be wired | -| Pod that should be skipped still gets wiring | The `inferencecache.io/skip-inject: "true"` annotation was missing or set on the Deployment, not the pod template | `kubectl get pod -o yaml` shows no `inferencecache.io/skip-inject` annotation | Add the annotation under `spec.template.metadata.annotations` of the Deployment and restart | - -The `inferencecache.io/skip-inject` annotation is the explicit escape hatch: any non-empty value other than the falsey set opts the pod out. The falsey set is what Go's `strconv.ParseBool` accepts as false (`false`/`FALSE`/`False`/`f`/`F`/`0`) plus the case-insensitive synonyms `no`/`off`/`disable`/`disabled`. Use the annotation for pods you've already pre-wired or that should run vanilla for a debugging experiment. A skipped pod keeps the original `skip-inject` annotation, gets `inferencecache.io/inject-skipped: "skip-inject-annotation"`, and does not get `inferencecache.io/injected-by`. +| Symptom | Cause | Fix | +|---|---|---| +| `MATCHED: 0` and no injection annotation | Selector and Pod labels differ. | Align the labels and recreate the Pod. | +| A matching Pod has no injection annotation | Admission failed open because of an invalid/colliding Pod shape or an unavailable managed Redis endpoint. | Read webhook logs and Pod Events, fix the reported shape, then recreate the Pod. | +| Engine crashes after successful injection | The runtime-owned image lacks a compatible LMCache client/API, or another engine startup requirement failed. | Inspect engine logs and use a compatible pinned image; CacheBackend does not replace it. | +| Multiple CacheBackends match one Pod | Selectors overlap; the lexicographically first CacheBackend wins. | Narrow selectors so every engine Pod has one owner. | +| Pod was relabeled after creation | Admission is CREATE-only. | Recreate the Pod. | +| Pod intentionally needs no cache injection | No explicit opt-out was set. | Put `inferencecache.io/skip-inject: "true"` on the Pod template and recreate it. | + +Legacy topology-less vLLM/IP binding remains implemented only for Phase 7 +compatibility tests. It is not a current sample or recommended production path. diff --git a/docs/concepts/cachebackend-engine-overrides.md b/docs/concepts/cachebackend-engine-overrides.md index 3e1a7420..c34b5ce3 100644 --- a/docs/concepts/cachebackend-engine-overrides.md +++ b/docs/concepts/cachebackend-engine-overrides.md @@ -1,300 +1,113 @@ # CacheBackend engine overrides -`CacheBackend.spec.integration.engineOverrides` is the in-between knob -between **"take the adapter's defaults"** and **"skip injection entirely."** -Use it when you want the LMCache integration on your engine pods, but need -to tune one knob — chunk size, log level, an extra vLLM flag — without -losing the rest of the canonical wiring. +`spec.integration.engineOverrides` amends the runtime-owned engine container +after a runtime adapter renders its canonical connector wire. Use typed +CacheBackend fields for cache topology and MP-server settings; use overrides +only for engine arguments or environment values that are not part of that +wire. -If you want no injection at all on a particular pod, set the -`inferencecache.io/skip-inject: "true"` annotation on the pod instead. -Overrides cannot un-wire the integration: every entry the adapter declares -as **reserved** (the args/env the integration cannot function without) is -hard-rejected at admission. +If a Pod should receive no cache injection, set +`inferencecache.io/skip-inject: "true"` on its template instead. ## The four primitives | Primitive | Semantics | |---|---| -| `env: [{name, value}, ...]` | Upsert by name. An override `name` matching an adapter-injected env replaces it; an override `name` the adapter did not inject is appended. An override `name` matching an env on your pod template that the adapter did NOT touch is a silent no-op (the override surface never mutates pod-template state the adapter did not invite the CR to touch). | -| `suppressEnv: [name, ...]` | Remove from the adapter's canonical env by name. Scoped to adapter-injected entries; suppressing a name the adapter did not contribute is a silent no-op. | -| `args: [...]` | For each entry, if its leading flag token (e.g. `--max-model-len`) matches an adapter-injected flag, replace the canonical entry; if it matches a user-template flag the adapter did not touch, the override is a silent no-op; if it matches neither, the entry is appended. Order is preserved. | -| `suppressArgs: [flag, ...]` | Remove from the adapter's canonical args by leading flag token. Scoped to adapter-injected entries. | +| `env` | Upsert an adapter-contributed environment variable by name, or append a new name. It does not rewrite unrelated Pod-template environment. | +| `suppressEnv` | Remove a non-reserved adapter-contributed environment variable. | +| `args` | Replace an adapter-contributed flag by leading token, or append a new flag absent from both the adapter and Pod template. | +| `suppressArgs` | Remove a non-reserved adapter-contributed flag. | -> **Mental model.** Canonical injection is what the adapter knows you -> need: the LMCache server URL, the connector config, the v1-engine flag. -> Overrides are what *you* know about *your* environment that the adapter -> doesn't: the chunk size your node memory tolerates, the log level your -> on-call playbook expects, the context length your tenant requested. +Entries that overlap the selected adapter's reserved arguments or environment +are rejected at CacheBackend admission. This prevents an override from silently +disconnecting the engine from the CacheBackend contract. -## Baseline — what the vLLM + LMCache adapter injects today +## Typed vLLM PodLocal MP baseline -For a `CacheBackend` named `qwen-demo` in namespace `default`, with no -`engineOverrides` block set, the controller's mutating Pod admission -webhook stamps the following on every matched engine container (the -`vllm` container, or the sole container if the pod has only one): +The current vLLM LMCache adapter injects: ```yaml -# vLLM container after webhook mutation — NO engineOverrides args: - # ... your pod-template args, unchanged ... - - --kv-transfer-config # RESERVED - - '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' + - --kv-transfer-config + - '{"kv_connector":"LMCacheMPConnector","kv_connector_module_path":"lmcache.integration.vllm.lmcache_mp_connector","kv_role":"kv_both","kv_connector_extra_config":{"lmcache.mp.host":"tcp://127.0.0.1","lmcache.mp.port":"5555"}}' + - --disable-hybrid-kv-cache-manager env: - - name: LMCACHE_REMOTE_URL # RESERVED - value: lm://qwen-demo.default.svc.cluster.local:65432 - - name: LMCACHE_REMOTE_SERDE # TUNABLE - value: naive - - name: LMCACHE_CHUNK_SIZE # TUNABLE - value: "256" - - name: LMCACHE_LOCAL_CPU # TUNABLE - value: "False" - - name: LMCACHE_MAX_LOCAL_CPU_SIZE # TUNABLE - value: "20" - - name: VLLM_USE_V1 # RESERVED - value: "1" - - name: INFERENCECACHE_FAIL_OPEN # RESERVED + - name: PYTHONHASHSEED + value: "0" + - name: INFERENCECACHE_FAIL_OPEN value: "true" ``` -`RESERVED` entries cannot be overridden or suppressed — admission -rejects the CR with a field-scoped error (see worked example 5). `TUNABLE` -entries are the ones the worked examples below amend. +It also injects the typed `lmcache-mp-server` native sidecar and shared +`/dev/shm`. These MP settings are not engineOverrides: -## Worked examples +- `spec.lmCache.chunkSizeTokens` controls chunk size; +- `spec.lmCache.podLocal.server.l1Capacity` controls per-Pod host capacity; +- `spec.lmCache.podLocal.server.image`, `port`, `maxWorkers`, and `resources` + control the injected server; and +- `spec.remoteStorage` explicitly selects an optional Redis L3. -Each example shows the CR fragment first, then the resulting engine -container args/env after webhook mutation. Lines marked `# ←` highlight -the difference from baseline. +The vLLM MP reserved set is: -### 1. Tune `LMCACHE_CHUNK_SIZE` for a memory-tight node +- arguments: `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager`; +- environment: `PYTHONHASHSEED`, `INFERENCECACHE_FAIL_OPEN`. -A node where the default 256-token chunks cost too much memory per replica. -Drop to 64. +SGLang has a different reserved set because its launch surface differs: +`--enable-lmcache`, `--lmcache-config-file`, `LMCACHE_USE_EXPERIMENTAL`, and +`INFERENCECACHE_FAIL_OPEN`. -```yaml -spec: - runtime: VLLM - integration: - engineOverrides: - env: - - name: LMCACHE_CHUNK_SIZE - value: "64" -``` +## Safe example -After webhook mutation: - -```yaml -env: - - name: LMCACHE_REMOTE_URL - value: lm://qwen-demo.default.svc.cluster.local:65432 - - name: LMCACHE_REMOTE_SERDE - value: naive - - name: LMCACHE_CHUNK_SIZE - value: "64" # ← was "256" - - name: LMCACHE_LOCAL_CPU - value: "False" - - name: LMCACHE_MAX_LOCAL_CPU_SIZE - value: "20" - - name: VLLM_USE_V1 - value: "1" - - name: INFERENCECACHE_FAIL_OPEN - value: "true" -``` - -### 2. Add a debug env the adapter doesn't inject - -Turn on LMCache verbose logging and pipe trace output to a known path so a -sidecar can collect it. Neither variable is in the adapter's canonical set -— both are appended. +This changes the typed MP chunk size and independently appends a logging value +to the engine container: ```yaml spec: runtime: VLLM + type: LMCache + lmCache: + topology: PodLocal + chunkSizeTokens: 128 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi integration: engineOverrides: env: - name: LMCACHE_LOG_LEVEL value: DEBUG - - name: LMCACHE_TRACE_FILE - value: /tmp/lmcache-trace.log -``` - -After webhook mutation (canonical block unchanged, two entries appended): - -```yaml -env: - - name: LMCACHE_REMOTE_URL - value: lm://qwen-demo.default.svc.cluster.local:65432 - - name: LMCACHE_REMOTE_SERDE - value: naive - - name: LMCACHE_CHUNK_SIZE - value: "256" - - name: LMCACHE_LOCAL_CPU - value: "False" - - name: LMCACHE_MAX_LOCAL_CPU_SIZE - value: "20" - - name: VLLM_USE_V1 - value: "1" - - name: INFERENCECACHE_FAIL_OPEN - value: "true" - - name: LMCACHE_LOG_LEVEL # ← appended - value: DEBUG - - name: LMCACHE_TRACE_FILE # ← appended - value: /tmp/lmcache-trace.log -``` - -### 3. Suppress the local CPU tier - -A topology where the engine never offloads to a local CPU memory tier (all -KV blocks live on the remote `lmcache-server`). The canonical defaults -already disable the tier (`LMCACHE_LOCAL_CPU="False"`); suppressing the two -variables removes them from the engine env entirely so reviewers and -audit tooling see the minimal set. - -```yaml -spec: - runtime: VLLM - integration: - engineOverrides: - suppressEnv: - - LMCACHE_LOCAL_CPU - - LMCACHE_MAX_LOCAL_CPU_SIZE -``` - -After webhook mutation (the two adapter-owned env entries are stripped): - -```yaml -env: - - name: LMCACHE_REMOTE_URL - value: lm://qwen-demo.default.svc.cluster.local:65432 - - name: LMCACHE_REMOTE_SERDE - value: naive - - name: LMCACHE_CHUNK_SIZE - value: "256" - # LMCACHE_LOCAL_CPU — suppressed - # LMCACHE_MAX_LOCAL_CPU_SIZE — suppressed - - name: VLLM_USE_V1 - value: "1" - - name: INFERENCECACHE_FAIL_OPEN - value: "true" -``` - -### 4. Append a vLLM flag the adapter doesn't inject - -Extend the engine's context window to 32 768 tokens. The adapter does -not inject `--max-model-len`, so — **assuming the engine pod template -does not already set `--max-model-len`** — the override is appended. -(If your pod template already carries the flag, an `args` override for -the same flag is a silent no-op; edit the pod template instead, since -`engineOverrides` only mutates adapter-contributed entries by design.) - -```yaml -spec: - runtime: VLLM - integration: - engineOverrides: args: - --max-model-len - "32768" ``` -After webhook mutation against a pod template with no -`--max-model-len` of its own (canonical `--kv-transfer-config` -preserved, two override args appended): - -```yaml -args: - # ... your pod-template args, unchanged ... - - --kv-transfer-config - - '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' - - --max-model-len # ← appended - - "32768" -``` - -### 5. Attempt to override `LMCACHE_REMOTE_URL` — hard-rejected at admission - -`LMCACHE_REMOTE_URL` is reserved by the vLLM runtime adapter: changing it -would silently re-point the engine at a different cache than the one the -CacheBackend reconciler stood up. Admission rejects the CR with a -field-scoped error that names both the offending env and the adapter, so -the operator gets the failure at `kubectl apply` rather than discovering a -silently-wrong wiring later. - -Concretely, take the paired sample at -`config/samples/cachebackend-with-override.yaml` and replace the -`engineOverrides` block on the `qwen-demo` CacheBackend with the -following: - -```yaml -spec: - runtime: VLLM - integration: - engineOverrides: - env: - - name: LMCACHE_REMOTE_URL - value: lm://my-other-cache.default.svc.cluster.local:65432 -``` - -Save the edited file as `bad-override.yaml` and apply: - -```text -$ kubectl apply -f bad-override.yaml -Error from server (Invalid): error when creating "bad-override.yaml": admission webhook "vcachebackend.inferencecache.io" denied the request: CacheBackend.inferencecache.io "qwen-demo" is invalid: spec.integration.engineOverrides.env[0].name: Forbidden: env "LMCACHE_REMOTE_URL" is reserved by the "vllm" runtime adapter and cannot be overridden or suppressed via spec.integration.engineOverrides; the adapter strictly requires this env for the integration to function -``` - -The same shape applies to `suppressEnv` overlap, `args` overlap (e.g. -`--kv-transfer-config`), and `suppressArgs` overlap — the error names the -field path, the offending token, and the adapter every time. - -## How to discover what's reserved - -Two surfaces today: - -- `kubectl explain cachebackend.spec.integration.engineOverrides` lists - the four primitive subfields; drill in with - `kubectl explain cachebackend.spec.integration.engineOverrides.env` - (and the other three) to read each primitive's per-field merge - semantics. -- The reserved list for the vLLM + LMCache adapter lives in the adapter - source: `internal/adapters/builtin/runtime/vllm_lmcache.go`'s `ReservedArgs()` and - `ReservedEnv()` methods. Each entry is commented with WHY it is - reserved. - -There is **no CLI surface today** for "show me adapter X's reserved -list" — the validator only surfaces the offending entry when a rejected -CR happens to overlap, never the full list. That is a real discoverability -gap worth a separate ticket; this doc names it so operators are not -surprised. - -## When NOT to use it +Trying to replace `PYTHONHASHSEED`, suppress +`--disable-hybrid-kv-cache-manager`, or replace `--kv-transfer-config` is +rejected with a field-scoped error naming the adapter and reserved token. -`engineOverrides` is for tuning **within** an integration. If you find -yourself wanting to override the wiring itself — +## Discoverability -- repoint the engine at a different cache (`LMCACHE_REMOTE_URL`), -- swap the connector (`--kv-transfer-config`), -- disable the v1 engine codepath the LMCache connector targets - (`VLLM_USE_V1`), +`kubectl explain cachebackend.spec.integration.engineOverrides` documents the +four primitives. The complete reserved lists currently live in the runtime +adapters under `internal/adapters/builtin/runtime`; there is no CLI command that +prints them. That is a discoverability gap, not permission to override the +connector wire. -— you probably want a different `CacheBackend` CR (e.g. one whose -reconciler stands up the cache server you actually want the engine to -talk to). Trying to express "switch between integrations" through an -override is a hard-reject at admission by design. The `External` backend -type is intended for the "engine should attach to a pre-existing cache -the controller does not manage" use case, but the engine-side wiring for -External is not in the default adapter set today — track its rollout -before reaching for it. +Legacy topology-less vLLM/IP overrides remain covered only by compatibility +tests until Phase 7. They are not a current tuning surface. -## See also +See also: -- [`docs/design/cachebackend-api.md`](../design/cachebackend-api.md) — - the ADR covering why the override surface has this shape and the - hard-reject-vs-warn admission posture. -- `docs/concepts/cachebackend-engine-binding.md` — companion concept doc - on how CacheBackend pods bind to engine pods via the engine-selector - label match. *(TODO: add the cross-link from the binding doc back to - this page once it lands.)* -- `config/samples/cachebackend-with-override.yaml` — a runnable paired - sample (CacheBackend + matching engine Deployment) that exercises a - small override block. +- [`cachebackend-api.md`](../design/cachebackend-api.md) +- [`cachebackend-engine-binding.md`](cachebackend-engine-binding.md) +- [`config/samples/cachebackend-with-override.yaml`](../../config/samples/cachebackend-with-override.yaml) diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 77e3f000..bd4f0325 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -2,6 +2,12 @@ Status: implemented · Tracks: InferenceCache tech spec §4.1 · API group: `inferencecache.io/v1alpha1` +> **Current production contract:** LMCache uses typed +> `spec.lmCache.topology: PodLocal` multiprocess wiring for both vLLM and +> SGLang, with optional Redis selected explicitly. Topology-less LMCacheServer, +> Mooncake, `lm://`, and IP-connector descriptions below are retained only as +> legacy compatibility/history until Phase 7; they are not recommended paths. + `CacheBackend` is the namespaced CRD that describes an engine-side cache implementation, an optional remote-storage tier, and the engine integration policy that should use them. Provider lifecycle belongs to storage-provider @@ -19,7 +25,7 @@ adapters; runtime adapters own engine Pod wiring only. The `v1alpha1` contract is pre-launch and explicitly unstable (see the carve-out paragraph below for the precise terms); after the v1beta1 promotion, new fields must be additive and tightening validation on existing fields requires a versioned migration path. -**Pre-launch carve-out (active until v1beta1).** The project is pre-launch and `v1alpha1` is explicitly unstable: where keeping an inert, unidiomatic, or operator-confusing field through to `v1beta1` would compound the cleanup work, a per-change waiver allows in-place removal during alpha. Each such removal is gated on (1) a locked design decision naming the field and the reason, (2) zero current consumers (no external operator manifests, no cross-component code), and (3) replacement of the operator-facing surface where one existed. Closed precedent: `CacheTenant.spec.quota.maxMemoryBytes` and `status.memoryUsed` removed (we cannot enforce per-tenant byte budgets on shared engines, and the underlying observation would be double-counted across tenants). The cluster-aggregate sibling `CacheIndex.status.tenants[].memoryUsed` has the same honesty problem (summing per-tenant memory across replicas on a shared engine double-counts the same bytes once per tenant), but because it is a published v1alpha1 *status* field it is **deprecated and zeroed in place** rather than removed: the controller stops populating it (always `0`) and operators are redirected to the per-replica `CacheIndex.status.replicas[].cacheMemoryBytes` (engine total per replica, honest at that altitude), while the field stays in the schema for wire/shape compatibility until its removal at v1beta1. Current applied removals: `CacheBackend.status.health` and the `CacheBackendHealth` enum removed in favour of the standard `status.conditions[Ready|Degraded|Progressing]` surface (the old `Degraded` health value is replaced by `Conditions[Degraded]`), which the new `Ready` printer column displays; and `CacheBackend.spec.storage{,.pvc}` + `status.capacity` removed — the `lm://` LMCache server we provision is in-memory, so a local PVC could not honestly back it, and durability is expressed as a backend choice (the Mooncake backend, now implemented — see [Mooncake provider configuration](#mooncake-provider-configuration)) rather than a generic volume knob (locked decision: `docs/design/lmcache-server-persistence.md`; replacement surface: backend-type selection). Once `v1beta1` is promoted, this carve-out is closed: subsequent breaking changes require a versioned migration. +**Pre-launch carve-out (active until v1beta1).** The project is pre-launch and `v1alpha1` is explicitly unstable: where keeping an inert, unidiomatic, or operator-confusing field through to `v1beta1` would compound the cleanup work, a per-change waiver allows in-place removal during alpha. Each such removal is gated on (1) a locked design decision naming the field and the reason, (2) zero current consumers (no external operator manifests, no cross-component code), and (3) replacement of the operator-facing surface where one existed. Closed precedent: `CacheTenant.spec.quota.maxMemoryBytes` and `status.memoryUsed` removed (we cannot enforce per-tenant byte budgets on shared engines, and the underlying observation would be double-counted across tenants). The cluster-aggregate sibling `CacheIndex.status.tenants[].memoryUsed` has the same honesty problem (summing per-tenant memory across replicas on a shared engine double-counts the same bytes once per tenant), but because it is a published v1alpha1 *status* field it is **deprecated and zeroed in place** rather than removed: the controller stops populating it (always `0`) and operators are redirected to the per-replica `CacheIndex.status.replicas[].cacheMemoryBytes` (engine total per replica, honest at that altitude), while the field stays in the schema for wire/shape compatibility until its removal at v1beta1. Current applied removals: `CacheBackend.status.health` and the `CacheBackendHealth` enum removed in favour of the standard `status.conditions[Ready|Degraded|Progressing]` surface (the old `Degraded` health value is replaced by `Conditions[Degraded]`), which the new `Ready` printer column displays; and `CacheBackend.spec.storage{,.pvc}` + `status.capacity` removed. The original rationale referenced the now-legacy in-memory `lm://` server and Mooncake provider; current durability/sharing is an explicit typed MP L3 choice, normally Redis (historical rationale: `docs/design/lmcache-server-persistence.md`). Once `v1beta1` is promoted, this carve-out is closed: subsequent breaking changes require a versioned migration. ## Cache hierarchy and ownership @@ -29,10 +35,20 @@ The canonical API assigns one architectural dimension to each field: spec: runtime: SGLang type: LMCache + integration: + role: ReadWrite lmCache: + topology: PodLocal chunkSizeTokens: 256 - hostMemory: - capacity: 32Gi + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:... + port: 5555 + l1Capacity: 32Gi + maxWorkers: 4 + resources: + requests: {cpu: "1", memory: 33Gi} + limits: {memory: 33Gi} remoteStorage: provider: Redis ownership: Managed @@ -60,13 +76,21 @@ spec: runtime: SGLang type: LMCache lmCache: - hostMemory: - capacity: 32Gi + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:... + port: 5555 + l1Capacity: 32Gi + maxWorkers: 4 + resources: + requests: {cpu: "1", memory: 33Gi} + limits: {memory: 33Gi} ``` -This requests `SGLang -> LMCache host memory` only. The controller creates no -provider Deployment or Service, and the engine adapter injects the node-local -LMCache MP worker without an L2 adapter. +This requests SGLang typed PodLocal MP with host-only L1. The controller creates +no remote-provider Deployment or Service; the webhook injects the MP server +native sidecar without an L3 adapter. Capability resolution is deliberately two-dimensional: @@ -79,10 +103,11 @@ engine-wire adapter storage-provider adapter +--------- optional Binding -------+ ``` -The provider adapter owns workload and Service rendering and emits a structured -binding (`lm`, `resp`, or `mooncakestore`). The engine adapter declares which -bindings it accepts. Admission rejects unsupported combinations before an -engine Pod is created. +The current Redis provider adapter owns workload and Service rendering and +emits a structured RESP binding. The engine adapter declares which bindings it +accepts. Admission rejects unsupported combinations before an engine Pod is +created. Legacy `lm` and `mooncakestore` bindings remain implemented only so +old alpha objects stay reconcilable until Phase 7. ### Cache type validation @@ -92,9 +117,9 @@ Mooncake is selected through `remoteStorage.provider`, and externally managed infrastructure through `remoteStorage.ownership`. The API server rejects the old `type: Mooncake` and `type: External` spellings before admission. -The canonical External and Mooncake examples are available in -[`config/samples/cachebackend-external.yaml`](../../config/samples/cachebackend-external.yaml) -and [`config/samples/cachebackend-mooncake.yaml`](../../config/samples/cachebackend-mooncake.yaml). +Current managed and external Redis examples are available in +[`config/samples/cachebackend-lmcache.yaml`](../../config/samples/cachebackend-lmcache.yaml) +and [`config/samples/cachebackend-external.yaml`](../../config/samples/cachebackend-external.yaml). ## Spec @@ -102,14 +127,14 @@ and [`config/samples/cachebackend-mooncake.yaml`](../../config/samples/cacheback |---|---|---| | `runtime` | enum | Required inference runtime: `VLLM` or `SGLang`. Values are case-sensitive. | | `type` | enum | Engine-side cache implementation: `LMCache` or `SGLangHiCache`. Defaults to `LMCache`. | -| `lmCache` | object | Typed LMCache engine configuration: chunk size, host-memory capacity, MP-worker image/port, and remote serde. | +| `lmCache` | object | Typed LMCache MP configuration: topology, chunk size, and PodLocal server image/port/L1/resources. | | `remoteStorage` | object | Optional remote tier. Omitting it means host-only and provisions no provider workload. | -| `remoteStorage.provider` | enum | `Redis`, `LMCacheServer`, or `Mooncake`. | +| `remoteStorage.provider` | enum | Current MP provider: `Redis`. `LMCacheServer` and `Mooncake` remain in the alpha schema only for legacy compatibility until Phase 7. | | `remoteStorage.ownership` | enum | `Managed` or `External`. | -| `remoteStorage.endpoint` | string | Required for `External`, rejected for `Managed`; managed endpoints are controller-observed in status. Bare `host:port` is portable across all providers. `LMCacheServer` also accepts `lm://host:port`, `Mooncake` also accepts `mooncakestore://host:port`, and `Redis` accepts only bare `host:port`. Every provider requires a numeric port in `1-65535`; admission rejects schemes belonging to another provider. | +| `remoteStorage.endpoint` | string | Required for `External`, rejected for `Managed`; managed endpoints are controller-observed in status. Current Redis requires bare `host:port`. Legacy providers retain their old scheme validation only while the compatibility implementation exists. | | `remoteStorage.redis` | object | Redis-owned image and resource configuration. | -| `remoteStorage.lmCacheServer` | object | Standalone LMCache-server-owned image, command, and resource configuration. | -| `remoteStorage.mooncake` | object | Mooncake-owned image, command, and resource configuration. | +| `remoteStorage.lmCacheServer` | object | Legacy IP compatibility field; not a current MP backend. Removed in Phase 7. | +| `remoteStorage.mooncake` | object | Legacy IP compatibility field; not a current MP backend. Removed in Phase 7. | | `observation` | object | Observation-owned `modelID` and `firstEventTimeout`. | | `deploymentKind` | enum | Managed workload kind: `Deployment` or `StatefulSet`. Defaults to `Deployment`. | | `replicas` | integer | Desired managed backend replicas. Defaults to `1`. Minimum `0`. See [Defaulting](#defaulting-mutating) for the interaction with `spec.autoscaling.minReplicas` (first-apply-only). | @@ -151,12 +176,12 @@ It intentionally does not expose `containers`; requiring users to provide contai ### Resources -Canonical resources place `corev1.ResourceRequirements` under the provider that -owns the workload: `remoteStorage.redis.resources`, -`remoteStorage.lmCacheServer.resources`, or -`remoteStorage.mooncake.resources`. The provider renderer deep-copies that -block onto its managed container. If the typed block is omitted, the provider -uses a bounded 4Gi request / 8Gi limit without persisting a default into the CR. +Current resources live with the workload owner: +`lmCache.podLocal.server.resources` for the MP native sidecar and +`remoteStorage.redis.resources` for managed Redis. Legacy +`remoteStorage.lmCacheServer.resources` and `remoteStorage.mooncake.resources` +remain renderer inputs only until Phase 7. Provider renderers deep-copy the +selected block onto their managed container. **Pass-through to the rendered container.** The provider adapter `DeepCopy`'s the selected typed resource block onto `Container.Resources`. The deep copy is @@ -243,13 +268,13 @@ SGLang supports two peer cache integrations: | Runtime/backend pair | Data plane | Controller-managed workload | |---|---|---| -| `(SGLang, LMCache)` without `remoteStorage` | Node-local LMCache MP worker, host-only | None | -| `(SGLang, LMCache)` with Managed Redis | Node-local LMCache MP worker with a shared Redis remote tier | Redis Deployment and Service | +| `(SGLang, LMCache)` without `remoteStorage` | PodLocal LMCache MP server, host-only | Native sidecar in each selected engine Pod | +| `(SGLang, LMCache)` with Managed Redis | PodLocal LMCache MP server with a shared Redis remote tier | Native sidecar plus Redis Deployment and Service | | `(sglang, SGLangHiCache)` | Native engine-local host cache | None | #### SGLang LMCache MP mode -> **SGLang drives LMCache in multiprocess (MP) mode (implemented, GPU-validated end to end).** Unlike vLLM, SGLang reads LMCache config from a **`--lmcache-config-file`** (carrying `mp_host`/`mp_port`), attaches to a **node-local MP worker** over ZMQ + a shared-memory data path, and offloads to a shared **L2 store** (the worker's `--l2-adapter`) — it does NOT use a cluster-reachable `lm://` server (`lm://` is not even a valid MP `--l2-adapter` type). So the `(sglang, LMCache)` data plane differs from vLLM's on **both** halves, and the sections below reflect that. Authoritative design + validation evidence: [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md). +> **SGLang drives LMCache in multiprocess mode (implemented and GPU-validated).** SGLang reads the generated client config through `--lmcache-config-file` and attaches to the PodLocal `lmcache server` over loopback plus shared memory. Optional Redis is an L3 adapter selected explicitly. It does not use the legacy cluster-reachable IP server. SGLang is the second runtime the cache plane supports (`spec.runtime: SGLang`, `spec.type: LMCache`; adapter at `internal/adapters/builtin/runtime`). Its engine @@ -260,15 +285,19 @@ Redis workload only when `spec.remoteStorage` explicitly selects > **Cluster prerequisite — Kubernetes ≥ 1.29 (REQUIRED for typed PodLocal LMCache).** The MP server is injected as a **native sidecar** — an `initContainers` entry with `restartPolicy: Always`, which K8s only understands from 1.29 (beta, on by default; stable 1.33). On an older cluster the apiserver does not recognize that field, so a typed SGLang or vLLM PodLocal engine pod fails admission (or the server degrades to a plain init container that exits before the engine starts). There is no in-webhook version gate today; operators using typed PodLocal LMCache must run 1.29+. -> **Two more caveats on the SGLang support surface** (details below): (1) server-derived `LookupRoute` with raw `token_ids`/`prompt_text` only hits when the server's single global `--engine-block-size` matches SGLang's page size (see the "Block-size alignment" note later in this section); gateways that send pre-computed `prefix_hash`/`block_hashes` are unaffected. (2) The `lmcache-kernel-check` init container is vLLM-only today (the SGLang adapter does not implement `InitContainerProvider`), so `EngineKernelsHealthy` is not published for SGLang pods. +> **Lookup caveat:** server-derived `LookupRoute` with raw +> `token_ids`/`prompt_text` only hits when the server's global +> `--engine-block-size` matches SGLang's page size. Gateways that send +> pre-computed hashes are unaffected. CacheBackend does not inspect or replace +> the engine image and does not add a package-verifier init container. The webhook renders the MP data plane on the SGLang engine pod. Alongside the -engine container it adds a **node-local MP-worker native sidecar** (an init -container with `restartPolicy: Always`) that writes the `--lmcache-config-file` -then runs the LMCache MP server on `127.0.0.1`. With a RESP binding it appends -`--l2-adapter` and offloads to Redis; without a binding it runs host-only. +engine container it adds a **PodLocal `lmcache-mp-server` native sidecar** (an +init container with `restartPolicy: Always`) that runs the supported +`lmcache server` entry point on `127.0.0.1` and writes the client configuration. +With a RESP binding it offloads to Redis; without a binding it runs host-only. `NVIDIA_VISIBLE_DEVICES=all` -lets the GPU-less sidecar CUDA-IPC the engine's GPU with no device-plugin +lets the GPU-less sidecar use CUDA-IPC with no device-plugin allocation, an `exec` startup-probe on the loopback ZMQ port gates the engine's start, and a shared `emptyDir` carries the config file. For `/dev/shm` (the L1 tier) it reuses the engine's own volume when the engine already mounts one (a duplicate @@ -287,7 +316,13 @@ reserved-names note below for the reuse/reject rules. On the engine container (n > > **Scope of what the adapter adds.** This is the engine image's own posture rather than something the adapter introduces: sglang images ship `NVIDIA_VISIBLE_DEVICES=all` in their `ENV`, and the device plugin overrides it only for containers that request a GPU (the engine gets a specific UUID; a request-less sidecar keeps the image default). The adapter sets it explicitly so the wire also works on a `workerImage` that lacks that default, instead of depending on an image side effect. Operators who need hard GPU isolation between tenants should not co-schedule those tenants on one node — the same guidance that applies to any CUDA-IPC sidecar. -**Names the MP wire reserves on the engine pod.** The init container `lmcache-mp-worker`, the volumes `lmcache-config` + `lmcache-dshm`, and the mount path `/etc/lmcache` are adapter-owned. If the pod already carries one of them and the adapter did not render it, admission **rejects the injection** — which the pod webhook turns into a fail-open admit, so the pod starts **un-wired** (no cache) rather than with its own container silently overwritten. The same applies when the engine mounts `/dev/shm` read-only or from a `configMap`/`secret`/`downwardAPI`/`projected` volume: the MP data path writes there, so it is rejected at admission instead of failing deep inside LMCache at runtime. Rename the colliding object (or drop the `readOnly`) to get the pod wired. Re-injecting an already-wired pod is **not** a collision — the adapter recognises its own worker and converges it on the current render. +**Names the MP wire reserves on the engine pod.** The init container +`lmcache-mp-server`, volumes `lmcache-mp-config` and `lmcache-mp-shm`, and mount +path `/var/run/inference-cache/lmcache` are adapter-owned. A foreign collision +rejects injection, which the pod webhook reports while admitting the pod +unwired under fail-open semantics. Re-injecting an operator-rendered pod is +idempotent. An incompatible existing `/dev/shm` mount is rejected rather than +failing later inside LMCache. The old lm:// `LMCACHE_REMOTE_URL` / serde / chunk-size / local-CPU env is **NOT** injected — SGLang MP mode ignores it. New manifests use typed @@ -295,10 +330,13 @@ The old lm:// `LMCACHE_REMOTE_URL` / serde / chunk-size / local-CPU env is | Field | Default | Bounds | Purpose | |---|---|---|---| -| `lmCache.chunkSizeTokens` | `256` | `>=1` | The worker's `--chunk-size` and config-file `chunk_size`. | -| `lmCache.hostMemory.capacity` | `4Gi` | positive quantity | Host-memory budget; rendered to the worker's whole-GiB L1 allocation. | -| `lmCache.workerPort` | `5555` | `1`–`65535` | Loopback ZMQ port used by the engine and worker. | -| `lmCache.workerImage` | engine image | — | Optional MP-worker image override. | +| `lmCache.chunkSizeTokens` | `256` | `>=1` | Server chunk size and client config. | +| `lmCache.topology` | required | `PodLocal` | `NodeLocal` is a future shape and is rejected. | +| `lmCache.podLocal.server.image` | required | digest-pinned reference | Independently owned LMCache server image; never copied from or into the engine image. | +| `lmCache.podLocal.server.port` | required | `1`–`65535` | Loopback MP port. | +| `lmCache.podLocal.server.l1Capacity` | required | positive quantity | Usable L1; `/dev/shm` and memory resources must cover this plus 1Gi. | +| `lmCache.podLocal.server.maxWorkers` | required | `>=1` | Server worker bound. | +| `lmCache.podLocal.server.resources` | required | validated K8s resources | Positive CPU request and sufficient memory request/limit. | Deliberately **not** injected for SGLang (a real engine difference, not an omission): `VLLM_USE_V1` (a vLLM-internal codepath with no SGLang analogue) and `PYTHONHASHSEED` (vLLM pins it to stabilise its builtin-`hash()`-seeded block-hash chain across TP workers; SGLang derives its prefix hash with `hashlib.sha256` over the token-id bytes, independent of `PYTHONHASHSEED`). @@ -306,7 +344,7 @@ Deliberately **not** injected for SGLang (a real engine difference, not an omiss **Reserved set** (`internal/adapters/builtin/runtime`): `ReservedArgs()` = `--enable-lmcache`, `--lmcache-config-file`; `ReservedEnv()` = `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. In MP mode the old lm:// `LMCACHE_REMOTE_URL` is neither injected nor reserved. `VLLM_USE_V1` / `PYTHONHASHSEED` are not reserved because they are never injected. -The two override surfaces are separate: `spec.lmCache` shapes the worker +The two override surfaces are separate: `spec.lmCache` shapes the server sidecar, while `spec.integration.engineOverrides` edits the engine container's args/env only. @@ -517,17 +555,21 @@ Extending the check to SGLang is a follow-up. server-side cache path): the kernel check catches the engine-side load cause that the round-trip probe cannot see. -### Mooncake provider configuration +### Legacy IP Mooncake provider configuration (compatibility only) + +> This section documents implementation retained until Phase 7 so old alpha +> objects remain diagnosable. Mooncake-through-LMCache/IP is not a current +> production path, is not a sample, and must not be translated to Redis +> automatically. Operators must explicitly choose typed host-only MP or Redis. `spec.remoteStorage.mooncake` selects the Mooncake provider adapter (`internal/adapters/builtin/storage/mooncake.go`) to reconcile the standalone **Mooncake master** workload. The vLLM runtime adapter separately wires engine pods to it through the LMCache remote-binding contract. Mooncake is -the durable / shared cache path — the backend-type expression of the persistence -decision in +historically represented a durable/shared cache path in [`docs/design/lmcache-server-persistence.md`](lmcache-server-persistence.md) -(the in-memory `lm://` lmcache-server is the simple default; Mooncake is the -scalable one — durability is a backend choice, not a generic volume knob). +(the old in-memory IP server and Mooncake mapping are both legacy; this text is +preserved only to explain the compatibility renderer). > **Operator requirement — the Mooncake master runs on the host network.** Unlike LMCache's `lm://` (one server, one port, one connection — a virtual ClusterIP suffices), Mooncake is a **peer-to-peer transfer-engine mesh**: the master on `:50051` returns only a directory pointer ("this block lives on node B"), and the engine then dials that node's real IP on a **dynamically negotiated port** to move the KV bytes. A ClusterIP Service forwards only the ports declared on it, and CNI overlay pod IPs are not reachable for the mesh — so the adapter renders the master with `hostNetwork: true` behind a **headless** Service (`clusterIP: None`), whose DNS name (published as `status.endpoint`) therefore resolves straight to the master's node IP with every port reachable. Consequences you must plan for: > @@ -747,7 +789,13 @@ Inference-cache has not been formally deployed, so this version does not ship a ### Engine-injection overrides (`spec.integration.engineOverrides`) -`spec.integration.engineOverrides` lets the operator amend the non-reserved args/env the pod-mutating webhook injects into the engine container — without forking an adapter. It is the user-facing seam that today's CPU-vLLM-with-LMCache use case and the SGLang+LMCache adapter reach to tune adapter-injected knobs (chunk size, max model length, serdes) that the canonical injection would otherwise hard-code. The reserved set (per locked decision #5/#6 below) makes this surface unsuitable for turning the integration *off*: operators who need to skip injection entirely on a pod should use the `inferencecache.io/skip-inject` annotation instead. +`spec.integration.engineOverrides` lets the operator amend non-reserved +engine-container args/env without forking an adapter. Current LMCache server +capacity, chunk size, port, image, and resources belong in typed +`spec.lmCache`; overrides are not a second configuration surface for those +fields. The reserved set makes this surface unsuitable for turning the +integration *off*: operators who need to skip injection entirely on a pod use +the `inferencecache.io/skip-inject` annotation instead. Shape, in `corev1` vocabulary: @@ -784,7 +832,7 @@ The legacy vLLM+LMCache adapter (`internal/adapters/builtin/runtime/vllm_lmcache - `ReservedArgs()`: `--kv-transfer-config` (the LMCache connector wiring). - `ReservedEnv()`: `VLLM_USE_V1` (selects the engine codepath the connector targets), `LMCACHE_REMOTE_URL` (the resolved cache endpoint), `INFERENCECACHE_FAIL_OPEN` (mirror of `spec.integration.failOpen` — overriding it would silently desync the pod from the CR contract), `PYTHONHASHSEED` (pins the deterministic `NONE_HASH` so LMCache reload matches under TP>1 — overriding or suppressing it silently 0-hits reload). -The same reserved set applies when the canonical vLLM/LMCache engine cache has +The same reserved set applies when a legacy vLLM/LMCache object has an External LMCacheServer binding or a Mooncake binding: the selected runtime adapter still runs the LMCache connector and varies only the structured binding's protocol and endpoint. Admission therefore rejects @@ -800,7 +848,10 @@ The typed PodLocal vLLM MP adapter reserves a narrower and different set: The SGLang+LMCache adapter (`internal/adapters/builtin/runtime`) reserves a **different** set, because SGLang's engine-side wire is the LMCache MP wire, not the `lm://` one (see [SGLang engine support](#sglang-engine-support)): `ReservedArgs()` = `--enable-lmcache`, `--lmcache-config-file`; `ReservedEnv()` = `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. Suppressing `--lmcache-config-file` un-wires MP mode (the engine aborts at startup without it), hence its reservation. In MP mode the lm:// `LMCACHE_REMOTE_URL` is neither injected nor reserved, and `VLLM_USE_V1` / `PYTHONHASHSEED` are never injected for SGLang. Reservation is per-adapter precisely so each engine guards only the flags/env its own integration cannot function without. -`LMCACHE_CHUNK_SIZE`, `LMCACHE_REMOTE_SERDE`, `LMCACHE_LOCAL_CPU`, `LMCACHE_MAX_LOCAL_CPU_SIZE` are deliberately NOT reserved — they are perf/mode tunables the operator may legitimately want to change. Canonical chunk size, serializer, and host-memory capacity use `spec.lmCache`; `engineOverrides.env` remains the engine-agnostic seam for explicit environment-level tuning. +`LMCACHE_CHUNK_SIZE`, `LMCACHE_REMOTE_SERDE`, `LMCACHE_LOCAL_CPU`, and +`LMCACHE_MAX_LOCAL_CPU_SIZE` belong only to the legacy IP adapter. They remain +unreserved while that compatibility adapter exists, but current MP manifests +must use the typed `spec.lmCache` hierarchy instead. #### Shape rationale (A vs. B) diff --git a/docs/design/kvevent-subscriber-wiring.md b/docs/design/kvevent-subscriber-wiring.md index 54007b3d..1cd2edb9 100644 --- a/docs/design/kvevent-subscriber-wiring.md +++ b/docs/design/kvevent-subscriber-wiring.md @@ -6,9 +6,9 @@ The substrate has two declarative halves: -* **Engine ↔ cache backend** — the C6 mutating Pod webhook injects `--kv-transfer-config` - and `LMCACHE_*` env vars onto the engine container when a `CacheBackend` claims the pod - (`internal/webhook/pod/podinjector.go`). +* **Engine ↔ cache backend** — the mutating Pod webhook injects the runtime- + specific typed LMCache MP connector plus a PodLocal native sidecar when a + `CacheBackend` claims the pod (`internal/webhook/pod/podinjector.go`). * **Engine ↔ policy server** — the engine publishes KV-cache events over ZMQ; a `kvevent-subscriber` process (`cmd/kvevent-subscriber`) consumes them and reports cache state to the policy server. @@ -47,7 +47,9 @@ Concretely: container spec (via their shared internal subscriber renderer — the KV-event stream is the engine's own ZMQ publisher, independent of the L2 store; each adapter pins its engine's `--hash-scheme` tag + ZMQ port); the reference adapter returns `(nil, nil)`. - External ownership stays on the runtime/cache adapter and can attach observation. + External Redis ownership stays on the runtime/cache adapter and can attach + observation. Legacy IP/Mooncake adapters retain observation behavior only + until Phase 7. * The Pod webhook (`internal/webhook/pod/podinjector.go`) calls `ObservationSidecar` right after `InjectEngineConfig`. A non-nil container is appended to `pod.Spec.Containers` (idempotent — skipped if a container by the well-known name is already present). Errors @@ -123,8 +125,8 @@ the prefix becoming unreachable and the cache plane must drop the routing hint promptly — the subscriber's default behavior, forwarding `BlockRemoved` as `PREFIX_EVICTED`, is exactly right. -When the engine is paired with a separate **L2 cache tier** (e.g. LMCache via -`--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1",...}'`) the +When the engine is paired with a separate **host-cache tier** (current typed +LMCache MP via `LMCacheMPConnector`) the semantics invert. LMCache retains the block after the engine offloads it from HBM, so the replica can still serve the prefix cheaply from the L2 tier. The vLLM-emitted `BlockRemoved` no longer means "the prefix is gone"; it means @@ -138,7 +140,7 @@ The subscriber tags each reported prefix with a **cache tier** (`PrefixEntry.tie see `grpc-contract.md`) derived from the engine's block lifecycle. The available signals are only `BlockStored` / `BlockRemoved` / `AllBlocksCleared`: **vLLM's KV-event channel announces the T1 (HBM) lifecycle but emits nothing when -LMCache offloads a block to L2** — the `LMCacheConnectorV1` offload is invisible +LMCache offloads a block to the host tier** — LMCache offload is invisible to the KV-event surface. So T2 is not *directly observable*; it is *inferred* from a T1 eviction combined with knowledge that an L2 tier is configured: @@ -181,13 +183,13 @@ left the entry stale at T1) and is kept for backward compatibility, but the signal it carries is unchanged. When set, a `BlockRemoved` becomes a T2 downgrade; when unset, it forwards `PREFIX_EVICTED`. `AllBlocksCleared` and `BlockStored` flow normally in both modes. The shared internal subscriber renderer -(`internal/adapters/builtin/runtime/subscriber.go`) — which the vLLM/LMCache, -vLLM/Mooncake, and SGLang/LMCache adapters all call — sets the flag **per +(`internal/adapters/builtin/runtime/subscriber.go`) — which the typed +vLLM/LMCache and SGLang/LMCache adapters call — sets the flag **per integration mode**, because the L2 tier is present only in one of them: -- **`Offload` (default):** the adapter wires the LMCache KV connector (pointed at - an `lm://` LMCache server or a `mooncakestore://` Mooncake store), so an L2 - tier retains the block after the engine offloads it — `BlockRemoved` means +- **`Offload` (default):** the adapter wires the LMCache MP connector to the + PodLocal server, so the host tier can retain the block after the engine + offloads it — `BlockRemoved` means "moved tiers," not "gone." The helper sets `--ignore-block-removed=true` so the hint is re-reported at T2 and ages out on its freshness TTL. - **`EventsOnly`:** no KV connector is injected, so there is **no** L2 tier diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index dbfd69a1..3e8b668f 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -360,8 +360,8 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 2 | Engine-neutral PodLocal MP server renderer | Phase 1 | complete | | 3 | Production-credible SGLang PodLocal MP baseline | Phase 2 | complete | | 4 | vLLM PodLocal MP | Phase 3 | complete | -| 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | not started | -| 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 finding | +| 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | complete | +| 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 and Phase 5 findings | | 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | not started | | 8 | NodeLocal shared MP server topology | Phases 3–4; does not block Phase 7 | not started | @@ -675,7 +675,7 @@ probe, so traffic tests waited for the engine health endpoint. ## Phase 5 — migration tooling and consumer migration -- **Status:** Not started +- **Status:** Complete (2026-08-11) - **Depends on:** Phase 4 ### Objective @@ -690,13 +690,39 @@ because Phase 0 found no external users or installed legacy objects. ### Deliverables -- [ ] Convert canonical samples, reference-stack manifests, support tables, CLI +- [x] Convert canonical samples, reference-stack manifests, support tables, CLI output, documentation, screenshots, and non-transition fixtures to MP. -- [ ] Remove language that presents the legacy LMCache server as a CPU profile +- [x] Remove language that presents the legacy LMCache server as a CPU profile or default backend. -- [ ] Reconfirm the zero-external-consumer assumption before removal. -- [ ] If external consumers appear, add inventory/doctor, dry-run conversion, - unmappable-field reporting, deprecation Events, and rollback guidance. +- [x] Reconfirm the zero-external-consumer assumption before removal, within the + evidence boundary recorded below. +- [x] Re-evaluate the conditional tooling trigger. No external consumer or + installed legacy-object evidence appeared, so migration tooling, + deprecation Events, and a compatibility gate were not activated. + +### Phase 5 inventory and disposition + +The repository-wide `rg` inventory was classified before editing: + +| Class | Findings | Disposition | +|---|---|---| +| Production/current consumers | Legacy LMCache samples (`cachebackend-lmcache*`, External, CPU override, paired/override), five recipes, flat SGLang samples, vLLM/SGLang reference manifests, quickstart/concepts/site pages, support tables, and the reference Helm values file. | Converted to typed PodLocal MP with explicit host-only or Redis semantics. The unvalidated Helm mapping, legacy CPU-only LMCache sample, and Mooncake sample were removed rather than translated inaccurately. | +| Legacy implementation for Phase 7 | Topology-less API fields/provider enums and CRD schema, LMCacheServer/Mooncake renderers, vLLM IP connector/wire helpers, endpoint parser, lifecycle/status code, and the doctor endpoint-scheme parser. | Retained unchanged so legacy alpha objects remain reconcilable until Phase 7. | +| Historical/migration documentation | This roadmap, the SGLang MP spike, LMCache-server persistence decision, and legacy portions of the API design. | Retained with explicit history/compatibility banners; current sections and links point to typed MP. | +| Intentional compatibility tests | Go tests for legacy render/admission behavior; C2/C6 scripts/workflows; legacy portions of default-install smoke. | Retained for Phase 7 safety, labelled legacy-only. C2/C6 scheduled triggers were removed; default-install smoke uses inline legacy fixtures instead of current samples. | + +No repository-owned screenshot asset contained a legacy deployment. CLI golden +output contained no legacy backend recommendation, so no output fixture changed; +the `doctor` `lm://` parsing branch is implementation compatibility for Phase 7. + +**External-consumer evidence boundary.** The Phase 5 repository inventory found +no cross-repository manifest, API client, generated consumer, or migration input, +and the Phase 0 owner audit remains zero for external consumers and installed +legacy objects. This phase did not query every OCI cluster or organization-wide +source repository, so the zero claim is limited to the repository evidence and +the recorded owner/Phase 0 confirmation. No contrary evidence appeared; adding +tooling without an input population would therefore create an unused migration +surface. Migration rules: @@ -710,17 +736,32 @@ Migration rules: ### Validation -- [ ] Repository search finds no repository-owned production LMCache workload +- [x] Repository search finds no repository-owned production LMCache workload still using IP, `lm://`, `LMCacheServer`, or flat SGLang MP fields. -- [ ] Migrated samples and reference manifests pass admission/default-install - smoke. -- [ ] Any newly discovered legacy object has an owner and explicit disposition. +- [x] `make verify-samples` admits every applicable migrated sample (25 passed, + 2 pre-existing explicit opt-outs, 0 failed); reference YAML parses, and + the typed vLLM/SGLang default-install smoke fixtures remain the current + admission path. The live kind default-install workflow was not run locally. +- [x] Every retained legacy reference is classified as Phase 7 implementation, + history/migration documentation, or intentional compatibility coverage. ### Exit criteria -- [ ] Every repository-owned LMCache workload uses MP. -- [ ] No migration silently changes cross-Pod sharing behavior. -- [ ] Conditional tooling, if activated, reports zero unknown legacy shapes. +- [x] Every repository-owned production/current LMCache workload uses typed MP. +- [x] No migration silently changes cross-Pod sharing behavior: each converted + object explicitly selects host-only or Redis, and ambiguous legacy + LMCacheServer/Mooncake examples were not auto-mapped. +- [x] Conditional tooling was not activated because the re-audit found no input + population or unknown legacy shape. + +Validation completed on 2026-08-11: `git diff --check`, `go test ./...`, +`make verify-samples`, shell syntax checks for the modified canaries/smoke, +reference-manifest YAML parsing, production/current negative searches, and +`make ci` all passed. `make ci` reported its optional golden-vector check as +skipped because the local Python environment lacked `xxhash`; the target itself +completed successfully. No Kubernetes cluster or GPU was required or used for +this repository-consumer migration, and the live kind default-install workflow +was not run locally. ## Phase 6 — reject new IP objects @@ -1075,7 +1116,7 @@ The migration is complete only when all of the following are true: `LMCACHE_REMOTE_URL`. - [x] Remote-L3 lifecycle events do not automatically roll MP engines. - [ ] Every old IP object has been migrated or intentionally deleted. -- [ ] Canonical samples, reference manifests, CLI output, and design documents +- [x] Canonical samples, reference manifests, CLI output, and design documents describe only the implemented MP behavior. - [x] NodeLocal, if enabled, guarantees same-node server selection and accurate engine coverage; otherwise it remains rejected rather than partially diff --git a/docs/design/lmcache-server-persistence.md b/docs/design/lmcache-server-persistence.md index 0747533e..7b49bece 100644 --- a/docs/design/lmcache-server-persistence.md +++ b/docs/design/lmcache-server-persistence.md @@ -2,6 +2,12 @@ Status: locked · Scope: managed-backend durability (`CacheBackend`) +> **Historical legacy-IP decision record.** This document explains why the old +> generic PVC surface was removed. Its LMCacheServer/Mooncake recommendation is +> superseded: current LMCache uses typed PodLocal MP with optional explicit +> Redis. Do not treat the providers below as current defaults and do not map +> them automatically to Redis; the operator must choose the desired L3 semantics. + ## Decision `CacheBackend.spec.storage` — and the nested `storage.pvc.*` plus the @@ -11,11 +17,11 @@ per-`CacheBackend` volume knob: - Omitting canonical `spec.remoteStorage` selects an engine-local host tier and provisions no provider workload. -- The managed **in-memory `lm://` LMCache server** +- Historically, the managed **in-memory `lm://` LMCache server** (`spec.remoteStorage.provider: LMCacheServer`) is the simple shared tier. It keeps KV in process memory; it is not durable and does not persist across pod restarts. -- The managed **Mooncake provider** +- Historically, the managed **Mooncake provider** (`spec.remoteStorage.provider: Mooncake`) is the durable / shared / scalable path: a network-addressable store the engine reaches over the `mooncakestore://` remote wire. See @@ -48,7 +54,7 @@ ClusterIP, engines-anywhere model. - `spec.storage{,.pvc}` + `status.capacity` were removed as a category error: the Kubernetes-side PVC plumbing could be provisioned, but could never honestly back the in-memory server. -- The recommended durable / shared topology is the **Mooncake backend**. Its +- The historical durable/shared recommendation was the **Mooncake backend**. Its managed workload lifecycle lives in the provider adapter (`internal/adapters/builtin/storage/mooncake.go`), while the vLLM runtime adapter (`internal/adapters/builtin/runtime/vllm_lmcache.go`) owns engine wiring. diff --git a/docs/design/sglang-lmcache-mp-mode.md b/docs/design/sglang-lmcache-mp-mode.md index 3c2d7446..ef8f24ce 100644 --- a/docs/design/sglang-lmcache-mp-mode.md +++ b/docs/design/sglang-lmcache-mp-mode.md @@ -2,6 +2,14 @@ Status: **implemented and GPU-validated** for SGLang (Phase 2, increments 1–2); increment 3 (operator surface + the remaining SPOF containment) is open — see [Phased delivery](#phased-delivery). Facts below are live-validated unless marked otherwise. · Supersedes the "mirror the vLLM+LMCache adapter" model in [cachebackend-api.md](cachebackend-api.md) SGLang section · Built-in adapters: `internal/adapters/builtin/runtime`; public contract: `pkg/adapters/runtime` +> **Implementation history, superseded for the current operator contract.** This +> document records the SGLang spike that established MP viability. Its flat +> worker fields, engine-image worker default, and predictions that vLLM MP was +> future work are historical. Current production behavior is the typed PodLocal +> API and common standalone-server renderer defined by +> [`lmcache-multiprocess-migration-roadmap.md`](lmcache-multiprocess-migration-roadmap.md) +> and [`cachebackend-api.md`](cachebackend-api.md). + **LMCache upstream now recommends multiprocess (MP) mode for *both* vLLM and SGLang** (its quickstart: MP is *"recommended"* for vLLM via `LMCacheMPConnector`, and *"the SGLang integration now defaults to MP mode"*). MP mode is a **node-local diff --git a/docs/quickstart.md b/docs/quickstart.md index a2a84cef..c45fa2a8 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -21,25 +21,32 @@ spec: engineSelector: matchLabels: app: my-engine # must match your engine pods' labels + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: modelID: Qwen/Qwen2.5-0.5B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: {} ``` That is the whole CacheBackend. The runtime/cache pair, matching labels, model -identity, and remote provider are explicit; `spec.replicas` defaults to `1`, -the readiness gate's `firstEventTimeout` defaults to `5m`, and -`integration.failOpen` defaults to `true`. - -For a managed LMCache provider, the server image resolves from -`spec.remoteStorage.lmCacheServer.image` first and the controller's -`--lmcache-server-image` flag second. The shipped Kustomize install sets that -flag to the documented `lmcache/standalone:v0.4.7` baseline; operators should -replace it with a digest compatible with the lmcache client in their engine -image. +identity, and MP server contract are explicit; `integration.role` defaults to +`ReadWrite`, the readiness gate's `firstEventTimeout` defaults to `5m`, and +`integration.failOpen` defaults to `true`. Omitting `remoteStorage` selects +host-only MP: L1 is per engine Pod and there is no cross-Pod sharing. Add an +explicit Redis L3 when sharing is required. > **One label does the binding.** The value under > `engineSelector.matchLabels` must also appear on your engine pods' @@ -47,23 +54,19 @@ image. > inject the cache wiring at pod CREATE. Drift them apart and the engine runs > uncached — `kubectl get cachebackend` then shows `MATCHED: 0`. -The CacheBackend on its own provisions the managed cache server. To get a -**working end-to-end setup** you also need engine pods carrying that label and -publishing KV events. Rather than hand-assemble that here, copy a runnable -recipe — the fastest is the CPU dev path, which needs no GPU: +The CacheBackend injects the MP server into matching engine Pods; it does not +own or replace their image. To get a **working end-to-end setup** you need an +engine image containing a compatible LMCache client plus Pods carrying that +label and publishing KV events. A paired shape is available at: ```bash -kubectl apply -f config/samples/recipe-cpu-dev.yaml +kubectl apply -f config/samples/cachebackend-with-engine.yaml ``` -That single file ships the CacheBackend above plus a matching tiny-model vLLM -engine Deployment, with the engine wired to the cache (KV offload/reuse) and the -backend producing `LookupRoute` hints. Acting on those hints to actually route -requests is the gateway's job, which integrates separately — so this recipe is -the cache half, not a full gateway round-trip. (On a cold cluster the first -engine pod can race ahead of the cache server's endpoint being published; if so, -wait for the endpoint and `kubectl rollout restart` the engine — see the comment -at the top of the recipe.) +That file ships the typed host-only CacheBackend plus a matching vLLM +Deployment. Repository admission validation does not prove the supplied engine +image contains the connector; normal engine startup is the authoritative check. +Acting on `LookupRoute` hints remains the gateway's responsibility. > **One install-time prerequisite for observability.** The piece that publishes > KV events — the `kvevent-subscriber` sidecar — is only auto-attached when the @@ -105,18 +108,18 @@ Once the backend is Ready and engine pods are bound, three things are live: lower time-to-first-token, less recompute. (inference-cache provides the hint; the gateway owns the routing decision.) - **KV reuse.** Matched engine pods get the LMCache wiring injected - automatically, so their KV cache is offloaded to and reused from the managed - cache backend instead of being recomputed per request. + automatically, so their KV cache is offloaded to the per-Pod MP server and + reused instead of being recomputed per request. - **Observability.** `kubectl get cachebackend` surfaces the live state: ``` $ kubectl get cachebackend - NAME TYPE READY MATCHED ENDPOINT PREFIXES LASTEVENT AGE - my-cache LMCache True 1 my-cache.default... 128 12s 3m + NAME TYPE READY MATCHED ENDPOINT PREFIXES LASTEVENT AGE + my-cache LMCache True 1 128 12s 3m ``` - `READY` flips to `True` only after the managed-readiness baseline - (pods Up, Service endpoints) **and** the KV-event gate (a real + `READY` flips to `True` only after the MP connector/server coverage baseline + **and** the KV-event gate (a real event observed, not merely the pod being reachable). When functional probing is enabled and not bypassed, a per-stage *failed* probe outcome additionally downgrades `Ready=False`. On a `/probe` @@ -140,7 +143,9 @@ Once the backend is Ready and engine pods are bound, three things are live: annotation. Any active gate that reports a per-stage failure can hold the backend at `Ready=False` with a stage-specific reason on `.status.conditions[]` — see [Troubleshooting](#troubleshooting). - `MATCHED` is the engine-pod count the selector binds, and + `ENDPOINT` is empty for host-only PodLocal MP and contains only a configured + remote L3 endpoint, never the loopback MP connector address. `MATCHED` is the + engine-pod count the selector binds, and `PREFIXES` / `LASTEVENT` show the cache actually receiving state. ## Next steps diff --git a/docs/reference-stack/GPU-RUNBOOK.md b/docs/reference-stack/GPU-RUNBOOK.md index fd2a1525..e98b5733 100644 --- a/docs/reference-stack/GPU-RUNBOOK.md +++ b/docs/reference-stack/GPU-RUNBOOK.md @@ -39,8 +39,8 @@ you need a bigger card or more cards (tensor-parallel). - **On a 48 GB L40S:** ~27 GB KV pool — comfortable, and leaves host headroom for LMCache offload. **This is the recommended PoC card.** -> **LMCache changes the GPU math only indirectly.** LMCache offloads KV blocks -> off the GPU into **host RAM / disk**, so it relieves GPU KV pressure but adds a +> **LMCache changes the GPU math only indirectly.** Typed PodLocal MP offloads +> KV blocks from the GPU into the server's **host-memory L1**, so it relieves GPU KV pressure but adds a > **host-memory** requirement (see §3). It does not reduce the weights footprint. --- @@ -72,9 +72,9 @@ Rules of thumb: | Resource | Reference (8B) | Why | |---|---|---| -| Host RAM | ≥ 32 GB free | `LMCACHE_MAX_LOCAL_CPU_SIZE=20` GiB CPU offload tier + OS/engine. Scale with the offload buffer. | -| `/dev/shm` | ≥ 8 GiB (set in `manifests/deployment.yaml`) | vLLM uses shared memory for tensor/IPC; small `/dev/shm` causes cryptic NCCL/loader hangs. | -| Local disk | model size × 1.5 + LMCache disk tier | HF weight cache + optional LMCache disk offload. ~30 GB for 8B; size up for 70B. | +| Host RAM | engine budget + MP L1 + headroom | The reference uses `l1Capacity: 4Gi`; the MP server request/limit and `/dev/shm` must cover L1 plus at least 1Gi. | +| `/dev/shm` | ≥ typed L1 + 1Gi | Shared by the engine and injected MP server; the reference uses 8Gi. | +| Local disk | model size × 1.5 | HF weight cache. The host-only reference does not claim a local-disk LMCache tier. | | Network | 100 Gb+ RDMA for multi-node | Only if you later shard across nodes; single-node TP uses NVLink. | | Driver/runtime | NVIDIA driver + Container Toolkit; `nvidia` default Docker runtime | So kind/OKE pods can request `nvidia.com/gpu`. | @@ -116,7 +116,9 @@ kubectl get nodes -o json | jq '.items[].status.allocatable["nvidia.com/gpu"]' kubectl create namespace cache-substrate kubectl -n cache-substrate create secret generic hf-token --from-literal=token="$HF_TOKEN" -# 2. Apply. For multi-card, bump replicas/GPU + add --tensor-parallel-size (see below). +# 2. Install inference-cache first, then apply. Replace the deliberately +# non-pullable engine-image placeholder before creating the Deployment. +kubectl apply -k ../../config/default kubectl apply -f manifests/namespace.yaml -f manifests/deployment.yaml -f manifests/service.yaml kubectl -n cache-substrate rollout status deploy/vllm-lmcache-llama-8b --timeout=20m ``` @@ -145,4 +147,4 @@ like"): subscribe with `scripts/kv_events_subscriber.py` and fire | Loads but low throughput / frequent recompute | KV pool too small | bigger card, raise `gpu_memory_utilization`, or lean on LMCache offload | | `tensor-parallel-size` mismatch / hang at startup | TP ≠ GPU count, or heads not divisible | set TP = `nvidia.com/gpu`; check head count divisibility | | NCCL / loader hang on multi-GPU | small `/dev/shm`, or no NVLink (multi-GPU VM) | raise `/dev/shm`; use a bare-metal NVLink shape for TP | -| Host OOM with LMCache enabled | `LMCACHE_MAX_LOCAL_CPU_SIZE` > free host RAM | lower the buffer or pick a higher-RAM shape | +| Host OOM with LMCache enabled | typed `l1Capacity` + 1Gi headroom exceeds the sidecar/pod memory budget | lower `l1Capacity` consistently or raise the MP-server request/limit and `/dev/shm` size | diff --git a/docs/reference-stack/README.md b/docs/reference-stack/README.md index 2758b196..45d2537e 100644 --- a/docs/reference-stack/README.md +++ b/docs/reference-stack/README.md @@ -1,122 +1,100 @@ -# vLLM (+ SGLang) + LMCache reference stack +# vLLM and SGLang typed LMCache MP reference stack -A reproducible reference deployment of **vLLM** serving a model with **LMCache** -as its KV-cache backend, with **KV-cache events published over ZMQ**. Use it to -verify cache-aware behaviour end to end: +This directory demonstrates the current inference-cache integration: a typed +`CacheBackend` selects an inference-engine pod and the operator injects an +LMCache multiprocess native sidecar plus the engine connector configuration. +An optional Redis tier provides explicit cross-Pod sharing. -- a **prefix-cache hit** on a repeated long prompt prefix (lower latency, prefill - skipped), and -- a live **KV-cache event stream** (`BlockStored` / `BlockRemoved` / - `AllBlocksCleared`) that a cache-aware router or controller can consume. +The repository does not own or replace the engine image. Use a digest-pinned +vLLM or SGLang image containing a connector/package compatible with the pinned +LMCache server. Normal engine startup is the authoritative compatibility check. -The manifests here are intentionally minimal and explicit so they can serve as a -starting template for your own automation (an operator, a Helm release, or plain -`kubectl apply`). - -> **Two engine references live here.** This page walks the **vLLM** path; the -> **SGLang** sibling (same LMCache backend + ZMQ event wire) is at -> [`manifests/sglang-lmcache/`](manifests/sglang-lmcache/) with its own README. - -> **You need an NVIDIA GPU** for the full stack — vLLM loads weights on CUDA and -> LMCache offloads KV from GPU memory. See [`GPU-RUNBOOK.md`](GPU-RUNBOOK.md) for -> how to size GPU memory and pick a card. If you only want to validate the event -> wiring and prefix-cache behaviour without a GPU, use the -> [CPU-only path](#cpu-only-local-check). +> The GPU manifests require Kubernetes 1.29 or later for native sidecars and an +> NVIDIA GPU. The CPU-only manifest validates engine prefix caching and KV-event +> decoding, but does not run the LMCache MP data plane. ## Layout -| Path | What | +| Path | Purpose | |---|---| -| [`VERSIONS.md`](VERSIONS.md) | Pinned images / models / chart. **Read first.** | -| [`GPU-RUNBOOK.md`](GPU-RUNBOOK.md) | GPU sizing (VRAM math), shape/card table, multi-card tensor-parallelism. | -| [`kind/cluster.yaml`](kind/cluster.yaml) | Local kind cluster (NodePorts for the API + ZMQ; the ZMQ NodePort is used by the vLLM path — the SGLang manifest deliberately doesn't node-expose ZMQ). | -| [`manifests/`](manifests/) | GPU reference Deployment + Service. | -| [`manifests/cpu-local/`](manifests/cpu-local/) | CPU variant (no LMCache): prefix-cache hit + KV events. | -| [`manifests/sglang-lmcache/`](manifests/sglang-lmcache/) | **SGLang** + LMCache reference (the second engine) — GPU; the hand-built template the `(sglang, LMCache)` adapter mirrors. See its [README](manifests/sglang-lmcache/README.md) for the event-wire scope, validation split, and caveats. | -| [`helm/values-reference.yaml`](helm/values-reference.yaml) | Upstream vLLM Production-Stack chart path (alternative to the raw manifests). | -| [`scripts/`](scripts/) | ZMQ event subscriber, prefix-cache-hit test, synthetic publisher, tests. | -| `captures/` | Where you save your event-stream sample and a cache-hit screenshot. | - -## Prerequisites +| [`VERSIONS.md`](VERSIONS.md) | Image and validation evidence; read before substituting engine images. | +| [`GPU-RUNBOOK.md`](GPU-RUNBOOK.md) | GPU sizing and operational notes. | +| [`manifests/deployment.yaml`](manifests/deployment.yaml) | vLLM + typed host-only LMCache MP. | +| [`manifests/sglang-lmcache/`](manifests/sglang-lmcache/) | SGLang + typed LMCache MP + explicit external Redis. | +| [`manifests/cpu-local/`](manifests/cpu-local/) | CPU-only engine/event check without LMCache. | +| [`scripts/`](scripts/) | Event subscriber, prefix-hit test, and compatibility canaries. | + +There is no Helm values reference in this phase. The repository has not +validated an upstream chart API that can faithfully express the operator-owned +native sidecar contract, so inventing a chart mapping would be unsafe. + +## Install the operator + +Install `config/default` (or the equivalent published release) before creating +the reference `CacheBackend`: ```bash -brew install kind # or your platform's installer; kubectl + helm also required -kind --version # >= v0.23 +kubectl apply -k config/default +kubectl -n inference-cache-system wait \ + --for=condition=Available deployment --all --timeout=180s ``` -For the GPU path you additionally need an NVIDIA GPU host (or a managed GPU -cluster), the NVIDIA Container Toolkit / device plugin, and — for gated models — -a Hugging Face token. - ---- +Use your normal release installation instead in a managed cluster. The +mutating webhook must be available before the engine Deployment is created; +otherwise admission fails open and the existing pod remains unwired until it is +recreated. -## Deploy and test on a GPU +## vLLM GPU path -> Size the GPU first with [`GPU-RUNBOOK.md`](GPU-RUNBOOK.md). The 8B reference -> model fits on a single 24 GB card. +The manifest contains a deliberately non-pullable engine-image placeholder. +Replace it with a compatible digest-pinned image; do not change the +`CacheBackend` server image to disguise an incompatible engine package. ```bash -# 1. Cluster + GPU. On the GPU host, install the NVIDIA Container Toolkit and set -# the nvidia runtime as Docker's default FIRST, then create the cluster: kind create cluster --name inference-cache-substrate --config kind/cluster.yaml helm repo add nvdp https://nvidia.github.io/k8s-device-plugin helm install nvdp nvdp/nvidia-device-plugin -n kube-system -kubectl get nodes -o json | jq '.items[].status.allocatable["nvidia.com/gpu"]' # expect "1" -# (Or use any managed GPU cluster that advertises nvidia.com/gpu — the manifests -# are identical; only the cluster differs.) -# 2. Pin the image to a real digest (the manifest ships a non-applyable placeholder -# on purpose — see VERSIONS.md), then create the HF token secret: -kubectl create namespace cache-substrate -kubectl -n cache-substrate create secret generic hf-token --from-literal=token="$HF_TOKEN" +kubectl apply -f manifests/namespace.yaml +kubectl -n cache-substrate create secret generic hf-token \ + --from-literal=token="$HF_TOKEN" -# 3. Deploy. -kubectl apply -f manifests/namespace.yaml -f manifests/deployment.yaml -f manifests/service.yaml +# Install the operator, replace the engine placeholder, then create the typed +# CacheBackend and matching Deployment from the same YAML stream. +kubectl apply -f manifests/deployment.yaml -f manifests/service.yaml kubectl -n cache-substrate rollout status deploy/vllm-lmcache-llama-8b --timeout=20m - -# 4. Subscribe to the KV-cache event stream and save a sample. -pip install -r scripts/requirements.txt -python scripts/kv_events_subscriber.py --endpoint tcp://localhost:30557 \ - --topic kv-events --max 200 --json | tee captures/kv-events-sample.jsonl - -# 5. Demonstrate the prefix-cache hit (run while the subscriber is watching). -./scripts/prefix_cache_hit_test.sh # save the output to captures/ ``` -### What success looks like +The vLLM reference is host-only: `status.endpoint` is intentionally empty. +For cross-Pod sharing, explicitly select Redis as shown by the SGLang reference +or [`config/samples/cachebackend-lmcache.yaml`](../../config/samples/cachebackend-lmcache.yaml). +Legacy `LMCacheServer` and Mooncake providers are not automatically translated +because doing so would silently change L3 and sharing semantics. -- **Prefix-cache hit:** request 2 (same long prefix) is faster than request 1, and - vLLM's `prefix_cache_hits` counter increases. -- **Event stream:** the subscriber prints `BlockStored` events (with block hashes) - during request 1; the saved sample contains metadata only — hashes and counts, - never prompt text or token content. +## Verify traffic and KV events -### Alternative: upstream Helm chart +```bash +pip install -r scripts/requirements.txt +python scripts/kv_events_subscriber.py \ + --endpoint tcp://localhost:30557 --topic kv-events --max 200 --json -[`helm/values-reference.yaml`](helm/values-reference.yaml) deploys the same stack -via the vLLM Production-Stack chart, if you prefer Helm over raw manifests. It -disables the chart's built-in router (this reference is about cache state and -events, not routing). +./scripts/prefix_cache_hit_test.sh +``` ---- +A successful run shows a prefix-cache counter increase on the repeated prefix +and `BlockStored` events. Event captures contain hashes and counts, not prompt +text. The Service deliberately does not expose MP control/data ports. -## CPU-only local check +## SGLang GPU path -You can exercise the **whole engine-config path without a GPU** — both a -prefix-cache hit and the KV-cache event stream. vLLM's v1 engine runs on CPU -(vLLM >= ~0.21) and the event publisher works there too; it is just slower and -has no LMCache offload. Uses a tiny model on vLLM's CPU build. +See [`manifests/sglang-lmcache/README.md`](manifests/sglang-lmcache/README.md). +That manifest uses the same typed `CacheBackend` contract and makes Redis an +explicit external L3 choice. -> **Verified** (vLLM 0.21.0 CPU image, arm64): cold request ~31s, warm -> same-prefix request ~1.4s, `vllm:prefix_cache_hits` incremented, and real -> `BlockStored` events were captured over ZMQ with token content redacted. It -> needs enough RAM — see the memory note in -> [`manifests/cpu-local/deployment.yaml`](manifests/cpu-local/deployment.yaml). +## CPU-only event check -> **Match the image to your host arch.** `manifests/cpu-local/deployment.yaml` -> defaults to the **`-arm64`** image tag. On x86_64 hosts, change it to -> `vllm/vllm-openai-cpu:latest-x86_64` first (the tags are arch-specific): -> `sed -i 's/latest-arm64/latest-x86_64/' manifests/cpu-local/deployment.yaml`. +This path has no LMCache offload and needs no GPU. Match the CPU image tag to +the host architecture before applying it. ```bash kind create cluster --name inference-cache-substrate --config kind/cluster.yaml @@ -128,118 +106,21 @@ python scripts/kv_events_subscriber.py --endpoint tcp://localhost:30557 --topic MODEL=Qwen/Qwen2.5-0.5B-Instruct ./scripts/prefix_cache_hit_test.sh ``` -### No image pull / no cluster? Validate the consumer with the synthetic publisher - -If you can't pull the image or run a cluster, you can still confirm the -event-decode + token-redaction path with the synthetic publisher — it emits -vLLM-shaped frames, no image required: +Without a cluster or image pull, validate only the consumer and redaction path: ```bash -pip install -r scripts/requirements.txt python scripts/kv_events_synthetic_publisher.py --bind 'tcp://*:5557' & python scripts/kv_events_subscriber.py --endpoint tcp://localhost:5557 --max 4 -python scripts/test_kv_events.py # asserts token_ids never surfaces; token_count kept -``` - -`test_kv_events.py` is the regression check for the decode + token-redaction -logic. It is run manually (the repo's CI is Go-only and has no Python step), so -run it after changing the subscriber. - ---- - -## CacheBackend reconciler canary (CPU) - -[`scripts/canary_c2_reconcile.sh`](scripts/canary_c2_reconcile.sh) is a GPU-free, -on-demand canary for the **C2 reconciler**: it brings up a kind cluster, runs the -controller, applies a typed `CacheBackend` with a managed LMCacheServer, and asserts -the controller stands up a healthy serving backend (Ready condition True, endpoint -published) and owner-ref garbage collection when the CR is deleted. It exercises -the reconciler against real pods — the gap the envtest unit tests can't cover. -An optional traffic block drives prefix traffic through the Service and asserts -an engine prefix-cache hit, but it is opt-in (`SKIP_TRAFFIC=0`) and requires a -separately wired engine — see the script for the port-forward target and -metric source. - -```bash -docs/reference-stack/scripts/canary_c2_reconcile.sh -``` - -Like the full-chain canary it is **on-demand**, not a blocking gate: it needs -Docker + kind and pulls the standalone LMCache server image. The default path -checks only the managed backend lifecycle, so it does not need an inference -engine or GPU. - -## In-cluster auto-attach (production path) - -When the controller is installed in a cluster **and the operator passes -`--kvevent-subscriber-image=` on the controller** (the -`subscriber-image` make target emits the well-known dev tag; pin to a -digest in production), the pod-mutating webhook auto-attaches the -`kvevent-subscriber` as a sidecar to every engine pod whose labels match a -`CacheBackend.spec.engineSelector` and whose backend sets -`spec.observation.modelID`. -The subscriber's identity flags (`--replica-id`, -`--tenant-id`, `--model-id`, `--hash-scheme`) are derived from the CR + pod -— no operator-supplied flags, no out-of-band `kubectl port-forward` + manual -binary launch on the demo path. - -The default install ships with the flag unset and therefore does not -auto-attach: a nonexistent image would put the sidecar container into -`ImagePullBackOff`, which would keep the engine pod from going Ready and -turn the cache into a serving dependency. Operators opt in by passing the -image once they have one ready to ship. - -The shape decision and rationale are in -[`docs/design/kvevent-subscriber-wiring.md`](../design/kvevent-subscriber-wiring.md); -the end-to-end auto-attach behaviour is gated by the webhook envtest -(`internal/webhook/pod/envtest_integration_test.go`), which boots a real -apiserver, installs the webhook, applies a `CacheBackend`, creates a labeled -engine pod, and asserts the persisted pod carries the `kvevent-subscriber` -container with flags derived from the CR. Run it locally with: - -```bash -KUBEBUILDER_ASSETS=$(make test-env | tail -1) go test ./internal/webhook/pod/... -``` - -## Full-chain binary canary - -[`scripts/canary_e2e.sh`](scripts/canary_e2e.sh) is a complementary GPU-free -canary that exercises the **subscriber binary's data path** end-to-end on the -host (no Kubernetes admission in the loop): CPU vLLM engine → `kvevent- -subscriber` → policy server → index. It drives prefix traffic and asserts -both an engine prefix-cache hit and that the server index populated -(`inferencecache_index_entries > 0`). Builds the binaries, manages the engine -container, cleans up after itself, exits non-zero on failure. - -Because this canary launches the engine in plain Docker (no K8s admission), -the subscriber is hand-launched on purpose — the binary's wire protocol is -what the test exercises. The in-cluster auto-attach path is covered by the -envtest gate above. - -The subscriber additionally scrapes the engine's Prometheus `/metrics` and emits -a per-replica `ReplicaStats` (`cacheMemoryBytes`, `hitRate`, `pressure`) on a -configurable tick (default `--stats-interval=10s`), so the policy server's -`/snapshot.replicas[]` (and the `CacheIndex.status.replicas[]` surface the -controller scrapes from it) populate alongside the prefix stream. Map -`cacheMemoryBytes` to the engine's KV cache by passing -`--engine-cache-size-bytes` (it is multiplied by the active `*_cache_usage_perc` -gauge); leave it `0` to publish `cacheMemoryBytes=0` and let the other fields -populate normally. - -If the engine serves more than one model on the same `/metrics` endpoint, pass -`--engine-model-name=` to filter by vLLM's `model_name` label — -that label tracks the engine's identifier, which is independent of the cache -plane's `--model-id` index key. Leave it empty when the engine serves one model -and you want the scraper to consume every series unfiltered. - -```bash -docs/reference-stack/scripts/canary_e2e.sh +python scripts/test_kv_events.py ``` -Same on-demand profile as the reconciler canary (Docker, vLLM CPU image, ~10+ GiB -Docker VM RAM). Run locally or wire into a scheduled/dispatch job. +## Legacy compatibility canaries ---- +`canary_c2_reconcile.sh` and `canary_c6_engine_wiring.sh` intentionally exercise +the legacy IP implementation retained until Phase 7. They are manual +compatibility tests, not current deployment references, and their workflows are +not scheduled. Current typed MP rendering and admission are covered by Go tests +and `default_install_smoke.sh`. ## Teardown diff --git a/docs/reference-stack/VERSIONS.md b/docs/reference-stack/VERSIONS.md index 420d0851..0961ba05 100644 --- a/docs/reference-stack/VERSIONS.md +++ b/docs/reference-stack/VERSIONS.md @@ -1,149 +1,50 @@ -# Pinned versions — vLLM (+ SGLang) + LMCache reference substrate +# Typed LMCache MP reference versions and evidence -Everything the reference stack depends on, pinned. Bump here first, re-validate -on a GPU host, then propagate to any automation that templates these manifests. +The reference manifests use only the typed PodLocal multiprocess data plane. +Engine and server images are independently owned: this repository injects the +connector configuration and native sidecar, but does not replace the inference +engine image or infer compatibility from an allowlist or annotation. A normal +engine startup is the authoritative package/connector compatibility check. -> **SGLang runs LMCache in MP mode (implemented, GPU-validated).** SGLang does not -> use the `lm://` lmcache-server model at all — it drives LMCache in **multiprocess -> (MP) mode**: config via the `--lmcache-config-file` flag, a **node-local MP worker** -> over `mp_host`/`mp_port`, and a shared **Redis L2** behind that worker (`lm://` is -> not a valid MP `--l2-adapter` type, so it cannot be reused here). The adapter renders -> exactly that. So the SGLang rows below pin **the engine/worker image tuple + Redis**, -> NOT an `lm://` server. Authoritative design + evidence: -> [`sglang-lmcache-mp-mode.md`](../design/sglang-lmcache-mp-mode.md). The vLLM rows are -> unaffected — `lm://` remains vLLM's shipped, supported path. +## Manifest pins -| Component | Pin | Where | Notes | -|---|---|---|---| -| vLLM + LMCache image | `lmcache/vllm-openai@sha256:` | `manifests/deployment.yaml`, `helm/values-reference.yaml` | Upstream ships LMCache pre-installed. Requires the vLLM **v1** engine (`VLLM_USE_V1=1`). The manifests ship a **non-applyable placeholder digest** — substitute a real one (below) before the GPU run. | -| Model | `meta-llama/Llama-3.1-8B-Instruct` | `manifests/deployment.yaml` | Gated on HF; needs `HF_TOKEN`. Small enough for a single A10/L40S-class GPU. Swap freely. | -| SGLang engine + MP worker | base `docker.io/lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8` + **lmcache 0.5.1** → derive + `@sha256:` | `manifests/sglang-lmcache/deployment.yaml`; the controller-rendered worker defaults to the **engine's own image** (`spec.lmCache.workerImage` overrides) | **GPU-validated tuple** (store→flush→retrieve reuses KV). The engine and the MP worker MUST run the same lmcache version — they speak the MP wire to each other — which is why the worker defaults to the engine image; overriding `workerImage` makes that alignment yours to maintain. Base `lmsysorg/sglang` does **not** bundle lmcache: derive an image (`pip install lmcache==0.5.1`) and pin its digest. **cu13 is load-bearing**: lmcache 0.5.1 needs `libcudart.so.13`, so a cu12 base fails to align, and stock `v0.5.1.post2-cu126` is too old to have `--enable-lmcache`. See [SGLang derived image reproducibility](#sglang-derived-image-reproducibility). GPU-only. | -| lmcache-server image | `lmcache/standalone:v0.4.7` → `@sha256:` | controller `--lmcache-server-image`; `spec.remoteStorage.lmCacheServer.image` overrides per backend | **vLLM only.** The shipped Kustomize install supplies this reproducible baseline; the Go binary has no baked-in image version. Select a server image compatible with the lmcache client in the operator-supplied engine image. **Not part of the SGLang wire in any form**: SGLang is MP-only, and `lm://` is not a valid MP `--l2-adapter` type, so it is not reachable behind the MP worker either — the SGLang shared tier is the Redis L2 row below. (An earlier revision of this file predicted the MP fix would reuse this server behind a per-node worker; GPU validation disproved that, and the prediction is retired rather than left to mislead.) | -| Redis remote store (SGLang) | `docker.io/library/redis:7.4-alpine` → `@sha256:` | `spec.remoteStorage.redis.image` and `manifests/sglang-lmcache/deployment.yaml` | Optional shared remote tier selected explicitly with `provider: Redis`, `ownership: Managed`. Production should pin an exact release or digest. Omitting `remoteStorage` leaves the SGLang LMCache worker host-only. | -| SGLang model | `meta-llama/Meta-Llama-3-8B-Instruct` | `manifests/sglang-lmcache/deployment.yaml` | Served model for the SGLang reference, kept equal to `config/samples/cachebackend-sglang.yaml`'s `observation.modelID` so the managed-path docs line up. Gated on HF; needs `HF_TOKEN`. Swap freely, but keep the engine `--model-path`, the CacheBackend `observation.modelID`, and request `model` identical. | -| CPU image | `vllm/vllm-openai-cpu:latest-{x86_64,arm64}` | `manifests/cpu-local/deployment.yaml` | vLLM's dedicated CPU build (arch-tagged). Runs the v1 engine on CPU (vLLM >= ~0.21), incl. the KV-event publisher. Verified on `0.21.0` (arm64): prefix-cache hit + real ZMQ events. Needs adequate RAM (CPU baseline ~5 GiB + KV). | -| CPU model | `Qwen/Qwen2.5-0.5B-Instruct` | `manifests/cpu-local/deployment.yaml` | Ungated, tiny, CPU-runnable. | -| kind | `>= v0.23` | local | `brew install kind`. Node image `kindest/node:v1.31.x`. | -| vLLM Production Stack chart | `vllm/vllm-stack` (chart `>= 0.1.6`) | `helm/values-reference.yaml` | Upstream "reference Helm chart". Pin the chart version at `helm install --version`. | -| NVIDIA k8s-device-plugin | `>= 0.15` | GPU host only | Only for the GPU-on-kind path. | - -## Digest-pin the GPU images before the GPU run - -`latest` is fine for a local CPU check but should not be used for a real GPU -deployment — it is not reproducible, and it is exactly the value any automation -templating these manifests would hard-code. Before -the GPU test/dev run: - -```bash -docker pull lmcache/vllm-openai:latest -docker inspect --format='{{index .RepoDigests 0}}' lmcache/vllm-openai:latest -# -> lmcache/vllm-openai@sha256:... put THIS in deployment.yaml + VERSIONS.md -``` +| Component | Reference value | Notes | +|---|---|---| +| LMCache standalone server | `docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13` | Pinned by the typed `CacheBackend`; runs `lmcache server`. This exact reference digest is structurally tested, while the GPU evidence below used the separately recorded validation digest. | +| vLLM engine | non-pullable all-zero placeholder | Replace with a digest-pinned image containing the LMCache MP connector/package. The repository deliberately supplies no default engine image. | +| SGLang engine | non-pullable all-zero placeholder | Replace with a digest-pinned SGLang image containing a compatible LMCache client. The repository deliberately supplies no default engine image. | +| Redis | `docker.io/library/redis:7.4-alpine` | Used only by the SGLang reference as an explicit external L3 choice. Digest-pin it for production. | +| vLLM model | `meta-llama/Llama-3.1-8B-Instruct` | Gated; keep the served model and `observation.modelID` aligned. | +| SGLang model | `meta-llama/Meta-Llama-3-8B-Instruct` | Gated; keep the served model, request model, and `observation.modelID` aligned. | +| CPU-only engine | `vllm/vllm-openai-cpu:latest-{x86_64,arm64}` | Event/prefix-cache check only; no LMCache MP data plane. Mutable development tag, not a production pin. | -The SGLang reference pins **two** images: (a) the **derived engine image** (which -the MP worker also runs by default), and (b) the **Redis L2** store. It does **not** -pin an lmcache-server — SGLang never dials one. +The standalone reference digest and the GPU-validation digest differ. Do not +interpret structural manifest coverage as a claim that this exact engine/server +tuple has completed the live GPU matrix. -> **Note — the reference manifest now matches this topology.** -> `manifests/sglang-lmcache/deployment.yaml` renders the MP topology: a **Redis L2** + -> the **MP-worker native sidecar** + the derived engine image (still a non-applyable -> placeholder digest — substitute your own build). It is **derived from the -> GPU-validated adapter render** (`internal/adapters/builtin/storage/redis.go` + -> the SGLang adapter) and structurally -> checked (`kubectl apply --dry-run=client`); re-run it on a GPU before treating it as golden. -> The pins below are authoritative for **both** the manifest and the controller-rendered -> managed path. +## Recorded live validation -(a) The **derived** SGLang engine image with the lmcache client baked in (the -base `lmsysorg/sglang` does not bundle it). This image is used **twice**: as the -engine, and — by default — as the MP worker (`spec.lmCache.workerImage` overrides -it), which is what keeps the two on the same lmcache version: +Phase 3 and Phase 4 ran on 2026-08-10/11 in the SJC development environment: -```bash -# Build a derived SGLang image with a version-aligned lmcache client. The -# lmcache version must satisfy BOTH alignment constraints: (1) ENGINE <-> MP WORKER -# -- the two speak the LMCache MP wire to each other, so they must run the same -# lmcache (trivially satisfied when the worker defaults to this same image), AND -# (2) its native CUDA kernels match the SGLang base image's CUDA runtime -- a -# CUDA-mismatched wheel loads but silently falls back to a slow non-native path, -# and the standalone reference has no lmcache-kernel-check init container to catch -# it (see "LMCache client kernels <-> engine-image CUDA / vLLM alignment" in -# docs/design/cachebackend-api.md). GPU-validated tuple: -# lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8 + lmcache 0.5.1 -- cu13 is -# load-bearing (0.5.1 links libcudart.so.13). NOTE: there is no lmcache-SERVER -# alignment constraint here; the shared L2 is Redis, which speaks RESP. -cat > Dockerfile.sglang-lmcache <<'EOF' -# Digest-pin the base too — this file is the pinning authority, and a moving tag -# would silently change the derived image's inputs on rebuild. Resolve the digest -# with: docker pull lmsysorg/sglang: && -# docker inspect --format='{{index .RepoDigests 0}}' lmsysorg/sglang: -# The GPU-validated tuple is the nightly-dev-cu13-20260711-7de33ce8 base + -# lmcache 0.5.1; resolve from that tag with the command above. -FROM lmsysorg/sglang@sha256: -RUN pip install --no-cache-dir lmcache==0.5.1 -EOF -docker build -f Dockerfile.sglang-lmcache -t myrepo/sglang-lmcache:pinned . -docker push myrepo/sglang-lmcache:pinned -# Read the pushed digest from the push output ("... digest: sha256:..."), or -# query the registry (local `docker inspect .RepoDigests` is often empty until -# the image is pulled back): -docker buildx imagetools inspect myrepo/sglang-lmcache:pinned --format '{{.Manifest.Digest}}' -# -> sha256:... use myrepo/sglang-lmcache@ in manifests/sglang-lmcache/deployment.yaml -``` - -## SGLang derived image reproducibility - -The SGLang engine row above now names a **GPU-validated `(sglang-tag, -lmcache-version)` tuple**; what is still yours to supply is the **digest of the -derived image you build from it**. (This section previously said no validated tuple -existed — true when it was written, before a GPU was available; the tuple below -replaces that placeholder.) Concretely: - -- **Pin the DERIVED image, not the upstream base.** `lmsysorg/sglang` does **not** - bundle the lmcache client; bake `pip install lmcache` into your own image — see the - build steps above. The **reference manifest** still ships a non-applyable - placeholder digest under an `example.invalid/sglang-lmcache` name (all-zero - `@sha256:`) — substitute your derived image's real digest before the GPU run. - (The manifest already renders the MP topology — Redis L2 + MP-worker sidecar; only - substituting your operator-built image digest remains. The controller-rendered - managed path does not read this manifest.) -- **RESOLVED — the concrete `(sglang-tag, lmcache-version)` tuple is - `(lmsysorg/sglang:nightly-dev-cu13-20260711-7de33ce8, lmcache 0.5.1)`**, validated - end-to-end on an A100: the MP worker registers the engine's KV cache over CUDA-IPC - and a flushed prompt is served back out of LMCache. Two constraints that tuple - encodes, both learned the hard way: **cu13 is load-bearing** (lmcache 0.5.1 links - `libcudart.so.13`, so a cu12 base mis-aligns at runtime), and the **stock - `v0.5.1.post2-cu126` tag is too old** — it predates `--enable-lmcache` entirely. - The validation installed lmcache with `pip` at pod start; a real deployment should - bake it into a derived image and pin THAT digest here (a runtime `pip install` is - not reproducible). The `@sha256:` digest is still yours to fill from your own - build. **The alignment that matters is engine ↔ MP worker (same lmcache, since they - speak the MP wire) plus lmcache ↔ CUDA runtime — NOT lmcache ↔ lmcache-server**: the - SGLang shared tier is Redis, which speaks RESP and carries no lmcache-version - constraint. Both constraints the tuple must - satisfy (engine ↔ worker lmcache parity, and lmcache kernels ↔ the SGLang base - image's CUDA runtime) are spelled out in the build steps above. -- **What IS validated without a GPU:** SGLang's exact event wire is covered by the - Go `internal/subscriber` SGLang test; the Python synthetic publisher covers only - the shared decode/redaction. See - [`manifests/sglang-lmcache/README.md`](manifests/sglang-lmcache/README.md). +| Path | Engine evidence | LMCache evidence | Result | +|---|---|---|---| +| SGLang TP=1 | `docker.io/lmsysorg/sglang@sha256:920df39109c60429b0a23eaacfd2786fcf1595c12f3ca4fc6e153b2abe34865f` (`0.5.13.post1-cu129`) | Client wheel 0.5.3 CUDA 12.9 via a test-only runtime-owner overlay; standalone `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b` | Host-only store/retrieve, bounded L1 eviction, events/status, and managed-Redis replacement-Pod retrieval passed. | +| vLLM TP=1/2 | `us-sanjose-1.ocir.io/idqj093njucb/vllm-openai@sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (`0.25.1`) | Client wheel 0.5.3 CUDA 12.9 via a test-only runtime-owner overlay; same validation standalone digest | Host-only TP=1/2 retrieval, events/status, shared-memory budget, native extension checks, and supplemental Redis replacement-Pod retrieval passed. | -## Why this image / engine combo +The runtime-owner wheel overlays were validation scaffolding, not authority for +`CacheBackend` to mutate an engine image. These records prove the controller +path with those exact test inputs; they are not universal image endorsements. -**vLLM path:** +## Operator compatibility rule -- vLLM + LMCache via their reference manifests. Upstream packages both in - `lmcache/vllm-openai`, so a single container runs the engine and the LMCache - connector in-process, not a sidecar. -- **vLLM v1 is required**: the KV-event publisher (`BlockStored` / `BlockRemoved` - / `AllBlocksCleared`) and the `LMCacheConnectorV1` connector both live on the - v1 engine. The image's `latest` tag assumes v1. +Before production rollout: -**SGLang path:** +1. Build or select the engine image in the inference-system release process. +2. Pin both engine and standalone-server images by digest. +3. Create the typed `CacheBackend` and let the webhook inject MP wiring. +4. Treat engine startup/readiness as the compatibility verdict, then run a + store, local-cache reset, and retrieve test on the target GPU/CUDA stack. -- SGLang loads the LMCache client in-process too, but turns it on with - `--enable-lmcache` + `LMCACHE_USE_EXPERIMENTAL=True` (no `--kv-transfer-config`, - no `VLLM_USE_V1`). There is no single upstream image bundling both, so the - engine image is a **derived** `lmsysorg/sglang` + `pip install lmcache` (above). - The KV-event publisher is SGLang's own `--kv-events-config` ZMQ scheme, wire- - compatible with vLLM's (see the SGLang manifest README). +Do not map legacy `LMCacheServer` or Mooncake objects to Redis automatically. +The operator must explicitly choose host-only MP or a supported L3 because the +choice changes cross-Pod sharing and persistence semantics. diff --git a/docs/reference-stack/helm/values-reference.yaml b/docs/reference-stack/helm/values-reference.yaml deleted file mode 100644 index 9d1cbf65..00000000 --- a/docs/reference-stack/helm/values-reference.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# Upstream "reference Helm chart" path (vLLM Production Stack). -# -# helm repo add vllm https://vllm-project.github.io/production-stack -# helm install vllm-substrate vllm/vllm-stack \ -# --version -n cache-substrate --create-namespace \ -# -f values-reference.yaml -# -# This is the upstream-supported way to deploy vLLM+LMCache and is useful as an -# alternative to, or cross-check of, the raw manifests (../manifests). The chart -# also ships a built-in router, which this reference disables — the goal here is -# cache state and events, not routing. -# -# Requires a GPU node advertising nvidia.com/gpu and an `hf-token` secret with -# key `token`. -# -# NOTE: field names below (extraArgs, extraPorts, lmcacheConfig, vllmConfig) -# follow the Production-Stack chart schema as of the pinned version — verify -# against `helm show values vllm/vllm-stack --version ` before relying on -# them; the chart's schema evolves. The hand-written ../manifests are the -# primary reference; this chart path is an optional alternative. -servingEngineSpec: - runtimeClassName: "" # set to "nvidia" if your cluster requires it - modelSpec: - - name: "llama8b" - repository: "lmcache/vllm-openai" - # PLACEHOLDER — replace with a pinned digest before any GPU deployment. Left - # non-resolvable on purpose so the values file can't run an unpinned image. - tag: "REPLACE_WITH_PINNED_DIGEST" # e.g. set repository to ...@sha256: per the chart's pinning support; see ../VERSIONS.md - modelURL: "meta-llama/Llama-3.1-8B-Instruct" - replicaCount: 1 - requestGPU: 1 - vllmConfig: - v1: 1 # required for KV events + LMCacheConnectorV1 - enablePrefixCaching: true - maxModelLen: 16384 - # The chart does not template --kv-events-config; inject it verbatim: - extraArgs: - - "--kv-events-config" - - '{"enable_kv_cache_events":true,"publisher":"zmq","endpoint":"tcp://*:5557","replay_endpoint":"tcp://*:5558","buffer_steps":10000,"topic":"kv-events"}' - lmcacheConfig: - enabled: true - cpuOffloadingBufferSize: "20" # GiB - hf_token: - secretName: "hf-token" - secretKey: "token" - # Expose the ZMQ PUB port on the engine pod/Service (chart default exposes 8000 only). - extraPorts: - - name: kv-events - containerPort: 5557 - - name: kv-replay - containerPort: 5558 - -# We do NOT use the chart's router — the gateway decides routing; inference-cache -# only describes cache state. -routerSpec: - enableRouter: false diff --git a/docs/reference-stack/manifests/deployment.yaml b/docs/reference-stack/manifests/deployment.yaml index c2b1cf8f..8cf59ac8 100644 --- a/docs/reference-stack/manifests/deployment.yaml +++ b/docs/reference-stack/manifests/deployment.yaml @@ -2,12 +2,44 @@ # # SPDX-License-Identifier: Apache-2.0 -# vLLM + LMCache reference backend (GPU). +# vLLM + typed PodLocal LMCache MP reference backend (GPU). # -# A minimal, explicit Deployment you can apply directly or use as a template for -# your own automation. Each flag is annotated below so it is clear what it does -# and why. The KV-cache event publisher is always enabled so a cache-aware router -# or controller can subscribe to the engine's cache state. +# The CacheBackend is the only LMCache enablement switch. The inference-cache +# webhook injects the MP server native sidecar and vLLM connector JSON into the +# matching engine Pod; the engine image stays runtime-owned. Kubernetes 1.29+ +# is required. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-lmcache-llama-8b + namespace: cache-substrate +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + app: vllm-lmcache-llama-8b + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + observation: + modelID: meta-llama/Llama-3.1-8B-Instruct +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -40,23 +72,12 @@ spec: args: - "--port=8000" - "--enable-prefix-caching" # on by default on v1; explicit for the reference - # LMCache connector: vLLM reads/writes KV through LMCache (kv_both). - - "--kv-transfer-config" - - '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' # KV-cache event publisher over ZMQ — the stream a cache-aware router # or controller subscribes to. Emits BlockStored / BlockRemoved / # AllBlocksCleared (msgpack). - "--kv-events-config" - '{"enable_kv_cache_events":true,"publisher":"zmq","endpoint":"tcp://*:5557","replay_endpoint":"tcp://*:5558","buffer_steps":10000,"topic":"kv-events"}' env: - - name: VLLM_USE_V1 # KV events + LMCacheConnectorV1 require the v1 engine - value: "1" - - name: LMCACHE_CHUNK_SIZE # LMCache chunk size (tokens) - value: "256" - - name: LMCACHE_LOCAL_CPU # Phase-1 reference: CPU offload tier - value: "True" - - name: LMCACHE_MAX_LOCAL_CPU_SIZE # GiB of CPU offload buffer - value: "20" - name: HF_TOKEN # gated model pull valueFrom: secretKeyRef: diff --git a/docs/reference-stack/manifests/sglang-lmcache/README.md b/docs/reference-stack/manifests/sglang-lmcache/README.md index 5bac6935..d101bd8e 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/README.md +++ b/docs/reference-stack/manifests/sglang-lmcache/README.md @@ -1,320 +1,104 @@ -# SGLang + LMCache (MP mode) reference stack - -The second-engine sibling of the top-level [vLLM + LMCache reference](../../README.md): -a **SGLang** deployment that publishes **KV-cache events over ZMQ** and **offloads -KV to a shared Redis L2** via LMCache **multiprocess (MP) mode**. It is the hand-built -reference the `(sglang, LMCache)` runtime adapter (`internal/adapters/builtin/runtime`) -mirrors: [`deployment.yaml`](deployment.yaml) stands up the same shape the adapter -auto-injects — a Redis L2 store, the engine with `--enable-lmcache` + -`--lmcache-config-file`, and a **node-local MP-worker native sidecar** that offloads -to that Redis. The engine image / `--model-path` / resources / `--kv-events-config` -are operator-owned scaffolding the adapter assumes is present, so the file as a whole -is **not** byte-for-byte adapter output. - -> **Validation status.** This manifest is **derived from the GPU-validated adapter -> render** (`internal/adapters/builtin/runtime/sglang_lmcache_wire.go` + -> `internal/adapters/builtin/storage/redis.go`; the controller-rendered managed path was -> validated store→flush→retrieve end-to-end in the MP-mode increment) and is -> **structurally checked** (`kubectl apply --dry-run=client`). It has **not** been -> independently re-run end-to-end on a GPU in this exact hand shape — run it on a GPU -> host (below) before treating it as a golden reference. All pins live in -> [`../../VERSIONS.md`](../../VERSIONS.md). - -> **Kubernetes ≥ 1.29 required.** The MP worker is a **native sidecar** (an -> `initContainers` entry with `restartPolicy: Always`), which older apiservers do not -> understand. - -## Why this exists (and what's already validated) - -SGLang adopted vLLM's KV-event wire wholesale: `--kv-events-config` drives a ZMQ -`ZmqEventPublisher` emitting the **same** msgspec `BlockStored` / `BlockRemoved` / -`AllBlocksCleared` event structs vLLM does (the batch envelope adds a trailing -`attn_dp_rank` the decoder ignores). Two consequences: - -1. **The event-decode path is engine-agnostic and already covered by tests.** The - shipped `kvevent-subscriber` decodes SGLang's stream unchanged; the only difference - is the `--hash-scheme=sglang` tag. The Go decoder is exercised against a synthetic - SGLang-shaped frame in `internal/subscriber/sglang_wire_test.go`, and the - cross-engine isolation (`hash_scheme` keeps SGLang and vLLM prefixes disjoint) in - `internal/index` (`TestNoCrossEngineFalseHitVLLMvsSGLang`). -2. **You can validate the wire off-GPU** (below): the Go test covers SGLang's exact - wire shape; the Python synthetic tooling covers the shared decode/redaction logic. - -What this reference adds on top of those tests is the **real engine → ZMQ events + -LMCache offload** path on a GPU: a live SGLang pod publishing real -`BlockStored`/`BlockRemoved` frames while the MP worker offloads evicted KV to Redis. -Extending that to **index + `LookupRoute`** additionally requires the cache plane -installed and a `CacheBackend` with `runtime: SGLang`, `type: LMCache`, a -Managed Redis `remoteStorage`, an `engineSelector` matching these pods, and -`observation.modelID` set. The controller then auto-attaches the -`kvevent-subscriber` sidecar (see -`config/samples/cachebackend-sglang.yaml` and the install docs). The standalone -manifest ships no event consumer, so on its own it shows the engine serving + the -publisher started + KV offloading to Redis, **not** a populated index. - -## How the MP data plane fits together - +# SGLang + typed LMCache multiprocess reference + +This manifest is the SGLang sibling of the vLLM reference. A typed +`CacheBackend` selects the engine pod and inference-cache injects: + +- the `lmcache-mp-server` native sidecar; +- a shared memory-backed L1 and generated MP configuration; +- SGLang's LMCache enablement/config-file arguments; and +- optional Redis L3 arguments from the explicit `remoteStorage` choice. + +The manifest does not hand-author those fields. This keeps the reference on the +same production renderer exercised by admission and controller tests. + +## Scope and prerequisites + +- Kubernetes 1.29 or later, because the MP server is a native sidecar. +- One NVIDIA GPU; this TP=1 reference has no supported CPU fallback. +- The inference-cache controller and mutating webhook installed first. +- A digest-pinned SGLang engine image containing an LMCache client compatible + with the pinned standalone server. The manifest's all-zero digest is + deliberately non-pullable. +- A Hugging Face token for the gated model. + +CacheBackend does not own or replace the engine image. It also does not use an +image allowlist, a capability annotation, or a verifier init container. Engine +startup is the authoritative connector/package compatibility check. + +## Topology + +```text +SGLang engine container + | + | loopback MP control + shared /dev/shm data + v +injected lmcache-mp-server native sidecar + | + | RESP, selected explicitly + v +external Redis Service in this manifest ``` - ┌─────────────────────── engine pod ───────────────────────┐ - │ [initContainers] │ - │ lmcache-mp-worker (native sidecar, restartPolicy:Always)│ - │ writes /etc/lmcache/config.yaml (mp_host/mp_port) │ - │ runs the LMCache MP server on 127.0.0.1:5555 │──resp──▶ redis-l2 - │ ▲ CUDA-IPC + /dev/shm (L1) │ (ClusterIP, - │ [containers] │ shared L2) - │ sglang --enable-lmcache --lmcache-config-file │ - │ --kv-events-config (ZMQ :5557) ────────────────────────┼──▶ kvevent-subscriber - └────────────────────────────────────────────────────────────┘ (managed path) -``` - -The engine dials the **local** worker over `mp_host`/`mp_port` (never the Redis -endpoint directly); the worker holds the L1 in `/dev/shm` and offloads its shared -tier to Redis over the `resp` `--l2-adapter`. `lm://` is **not** a valid MP -`--l2-adapter` type, which is why the shared tier is Redis, not the standalone -`lmcache-server` the vLLM path uses. - -## Engine-side differences from the vLLM reference - -| | vLLM | SGLang | -|---|---|---| -| LMCache on | `--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1",…}'` | `--enable-lmcache` (bare flag) + `LMCACHE_USE_EXPERIMENTAL=True` | -| LMCache config source | `LMCACHE_*` env (`LMCACHE_REMOTE_URL`, …) | **`--lmcache-config-file`** (written by the MP worker); the `LMCACHE_*` env is ignored | -| Shared tier | standalone `lm://` `lmcache-server` | **Redis L2** behind a node-local **MP worker** (`resp` `--l2-adapter`) | -| vLLM-only env | `VLLM_USE_V1=1`, `PYTHONHASHSEED=0` | *(neither — no v1 codepath; SGLang sha256-hashes, independent of `PYTHONHASHSEED`)* | -| Default HTTP port | 8000 | 30000 | -| KV-event wire | ZMQ `BlockStored`/`BlockRemoved`/`AllBlocksCleared` | same event structs; batch envelope adds a trailing `attn_dp_rank` the decoder ignores | -## Deploy and test on a GPU +Redis is an explicit operator choice. Omitting `remoteStorage` produces a +host-only MP backend. A legacy LMCacheServer or Mooncake configuration is not +silently converted to Redis because that would change cross-Pod/L3 semantics. -> Needs an NVIDIA GPU host (or a managed GPU cluster advertising `nvidia.com/gpu`) -> and a Hugging Face token for the gated reference model. Size the GPU the same way -> as the vLLM path — see [`../../GPU-RUNBOOK.md`](../../GPU-RUNBOOK.md); the 8B -> reference model fits on a single 24 GB card. +## Deploy -This manifest is a **standalone** reference showing the hand-built shape the adapter -mirrors, with **no controller or `CacheBackend` in the loop** — the engine reads its -MP config from the worker's `--lmcache-config-file`, and the worker offloads to the -in-namespace `redis-l2` Service. - -Run the commands below from this directory -(`docs/reference-stack/manifests/sglang-lmcache/`) — the relative paths -(`../../kind/cluster.yaml`, `deployment.yaml`) assume it. +From `docs/reference-stack`: ```bash -# 1. Cluster + GPU device plugin (identical to the vLLM path). -kind create cluster --name inference-cache-substrate --config ../../kind/cluster.yaml -helm repo add nvdp https://nvidia.github.io/k8s-device-plugin -helm install nvdp nvdp/nvidia-device-plugin -n kube-system - -# 2. Fix the SGLang image in deployment.yaml: replace the ENTIRE placeholder -# reference `example.invalid/sglang-lmcache@sha256:0000...` (in BOTH the -# lmcache-mp-worker init container and the sglang engine container) with your -# real DERIVED image (repo AND digest) — the base sglang image does not bundle -# the lmcache client, so you must build one (`pip install lmcache==0.5.1` onto the -# GPU-validated cu13 base). See the deployment.yaml header + VERSIONS.md. Redis is -# a normal pullable tag (digest-pin for production). Then create the namespace + -# HF token secret (idempotent so the runbook re-runs cleanly). -kubectl create namespace cache-substrate --dry-run=client -o yaml | kubectl apply -f - -kubectl -n cache-substrate create secret generic hf-token --from-literal=token="$HF_TOKEN" \ - --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f manifests/namespace.yaml +kubectl -n cache-substrate create secret generic hf-token \ + --from-literal=token="$HF_TOKEN" -# 3. Deploy the Redis L2 + the SGLang engine (with its MP-worker sidecar). -# Wait for BOTH rollouts. The engine rollout only goes green once the MP worker's -# startupProbe passes (the ZMQ server is listening on 127.0.0.1:5555) — SGLang has -# no cacheless fallback while --enable-lmcache is on, so the worker is a serving -# prerequisite, not a soft dependency. -kubectl apply -f deployment.yaml +# Install inference-cache first, replace the engine placeholder digest in the +# manifest, then create Redis, CacheBackend, and the matching engine Deployment. +kubectl apply -f manifests/sglang-lmcache/deployment.yaml kubectl -n cache-substrate rollout status deploy/redis-l2 --timeout=5m -kubectl -n cache-substrate rollout status deploy/sglang-lmcache-llama-8b --timeout=20m - -# 4. Drive the SAME long-prefix prompt twice (first warms, second reuses). This -# exercises the engine, triggers its KV-event publisher on :5557, and — because -# MP mode is write-through — offloads stored KV to Redis immediately (not only on -# HBM eviction; see the DBSIZE check below). Swap the `prefix = ...` line -# for a REAL prompt long enough to span several KV blocks (>> the engine's -# --page-size in tokens); a short literal won't reliably produce BlockStored / -# prefix reuse. -BODY="$(python3 - <<'PY' -import json -prefix = "You are a helpful assistant. " * 200 # ~1k+ tokens of shared prefix -print(json.dumps({"model": "meta-llama/Meta-Llama-3-8B-Instruct", - "messages": [{"role": "user", "content": prefix}], - "max_tokens": 16, "temperature": 0})) -PY -)" -kubectl -n cache-substrate port-forward svc/sglang-lmcache-llama-8b 30000:30000 & -pf=$!; trap 'kill "$pf" 2>/dev/null' EXIT # clean up the port-forward (frees :30000) -ready="" -for _ in $(seq 60); do - kill -0 "$pf" 2>/dev/null || { echo "FAIL: port-forward exited early (is the pod running?)"; exit 1; } - curl -sf localhost:30000/health >/dev/null 2>&1 && { ready=1; break; } - sleep 2 -done -[ -n "$ready" ] || { echo "FAIL: engine /health not ready after ~120s"; exit 1; } -rc=0 -for i in 1 2; do - # -sfS: fail (non-zero) on HTTP 4xx/5xx so a rejected request doesn't look served. - curl -sfS localhost:30000/v1/chat/completions -H 'content-type: application/json' -d "$BODY" >/dev/null \ - || { echo "FAIL: request $i was not served (HTTP error)"; rc=1; break; } -done -kill "$pf" 2>/dev/null; trap - EXIT -[ "$rc" -eq 0 ] || exit 1 +kubectl -n cache-substrate rollout status \ + deploy/sglang-lmcache-llama-8b --timeout=20m ``` -### What success looks like (standalone) - -- **The engine serves both requests AND its KV-event publisher started.** Assert the - publisher concretely from the engine logs (so a mis-configured `--kv-events-config` - fails the check rather than silently passing): +Confirm admission injected the sidecar and connector wiring: - ```bash - # Match a publisher-STARTUP line, not a config/arg echo: SGLang echoes - # "--kv-events-config"/"kv-events" at boot, so grepping for "kv-events" would - # false-pass on the echo alone. Adjust the pattern to your SGLang build's phrasing - # (confirm against a real run's logs the first time). - kubectl -n cache-substrate logs deploy/sglang-lmcache-llama-8b -c sglang \ - | grep -iE 'zmq.*publish|publisher thread|start(ing|ed).*publisher' \ - || { echo "FAIL: KV-event publisher did not start (check --kv-events-config)"; exit 1; } - ``` - -- **KV *written* to Redis (write-through).** LMCache MP mode is write-through: a - stored prefix lands in the L2 immediately, not only on HBM eviction (GPU-validated — - a 3760-token prompt took Redis `DBSIZE` 0→14, i.e. 14 chunks of the 256-token - `chunk_size`). This proves the offload **write** path — assert the keyspace is - non-empty after the requests above: - - ```bash - # dbsize > 0 proves the MP worker wrote KV to redis-l2 over `resp`. - n=$(kubectl -n cache-substrate exec deploy/redis-l2 -c redis-l2 -- redis-cli dbsize | tr -dc '0-9') - [ "${n:-0}" -gt 0 ] || { echo "FAIL: Redis L2 empty after requests — offload not happening"; exit 1; } - echo "Redis L2 holds $n KV chunks" - ``` - -- **KV *reloaded* from Redis (the read path).** `DBSIZE > 0` only proves writes — a - repeat request could still be served from the engine's own GPU radix / L1 cache - without touching L2. To prove **retrieval**, flush the engine's local cache first, - then re-request, and look for the worker's reload signal (this is the - store→flush→retrieve cycle the managed path was GPU-validated on): - - ```bash - ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) # only count retrievals logged AFTER this point - kubectl -n cache-substrate port-forward svc/sglang-lmcache-llama-8b 30000:30000 & pf=$!; sleep 3 - curl -sfS -X POST localhost:30000/flush_cache >/dev/null \ - || { kill "$pf" 2>/dev/null; echo "FAIL: flush_cache request failed"; exit 1; } # clear GPU radix cache - curl -sfS localhost:30000/v1/chat/completions -H 'content-type: application/json' -d "$BODY" >/dev/null \ - || { kill "$pf" 2>/dev/null; echo "FAIL: post-flush request not served"; exit 1; } - kill "$pf" 2>/dev/null - # A retrieval logged AFTER the flush (scoped by --since-time so a stale warm-up - # retrieval can't false-pass) proves the L1/L2 reload: - kubectl -n cache-substrate logs deploy/sglang-lmcache-llama-8b -c lmcache-mp-worker --since-time="$ts" \ - | grep -iE 'Retrieved [0-9]+ tokens|Prefetch .*L2' \ - || { echo "FAIL: no L2 retrieval after flush — reload path not working"; exit 1; } - ``` - - (Full frame→index→`LookupRoute` verification needs the **managed path** below; the - standalone manifest ships no event consumer, so it never populates the index.) - -- **Privacy boundary — the RAW ZMQ frames carry `token_ids`.** Both vLLM's and - SGLang's `BlockStored` wire includes the block's token ids; the "metadata-only, - never token content" guarantee is about what the IC `kvevent-subscriber` *reports to - the policy server* — it hashes the token_ids in-pod into the content fingerprint and - forwards only hashes + counts (over `127.0.0.1`). That report-level guarantee holds - regardless of how the publisher binds. The **raw ZMQ frames** are a separate matter: - the manifest binds `tcp://*:5557` (the operator contract in - [`docs/design/cachebackend-api.md`](../../../design/cachebackend-api.md), kept - identical to the vLLM reference), so the raw, token-bearing frames **are** reachable - by any in-cluster pod that knows this pod's IP. Two things bound that exposure: the - Service deliberately exposes **only** the HTTP API (never `:5557`), and — if - in-cluster raw-frame access is a concern — you can add a **NetworkPolicy** restricting - `:5557` to this pod (the in-pod subscriber reaches the publisher over `127.0.0.1`, so - it is unaffected). If you must inspect the raw stream during dev, `kubectl - port-forward` `:5557` yourself rather than adding it to the Service. - -- **Why the two `scripts/` helpers don't verify SGLang here:** - `scripts/kv_events_subscriber.py` decodes only vLLM's 2-tuple synthetic frames (it - would print `UNDECODED` on SGLang's 3-tuple), and `scripts/prefix_cache_hit_test.sh` - reads vLLM's `vllm:prefix_cache_hits` counter SGLang doesn't emit (use it only as a - request driver). The shipped live SGLang consumer is the managed Go sidecar below. - -### The managed path (what the adapter automates) - -In a real install you do **not** hand-write this manifest. You create a -`CacheBackend` with `runtime: SGLang`, `type: LMCache`, and a Managed Redis -`remoteStorage` (see -[`config/samples/cachebackend-sglang.yaml`](../../../../config/samples/cachebackend-sglang.yaml)) -whose `engineSelector` matches your SGLang pods, and the controller renders the **Redis -L2** store, injects the **MP-worker sidecar + `--enable-lmcache` + `--lmcache-config-file`** -onto the engine pod, and — with `--kvevent-subscriber-image` set — auto-attaches the Go -`kvevent-subscriber` sidecar (tagged `--hash-scheme=sglang`) that reports to the index, -enabling `LookupRoute` to return SGLang replicas (never a vLLM replica on the same -prefix bytes — the `hash_scheme` tag keeps them disjoint). Two caveats the manual -manifest sidesteps but the managed path must honor: the CacheBackend must live in the -**engine pods' namespace** (the Pod webhook matches per-namespace), and because -injection is **create-time only**, the CacheBackend's `status.endpoint` (the Redis L2 -address) must be **published** before the engine pod is created, or the pod admits -unwired and must be recreated. (The precondition is specifically `status.endpoint`, -**not** `Ready` — managed `Ready` is gated on the first KV event observed *from these -very pods*, so waiting for `Ready` first would be circular.) The served model, the -CacheBackend's `observation.modelID`, and the request's `model` must all agree, or -the index keys per-model and `LookupRoute` silently misses. **Block-size caveat:** for -raw-`token_ids`/`prompt_text` lookups the server fingerprints at its single global -`--engine-block-size` (default 16, vLLM's); SGLang's page size (e.g. 64) must match it, -or gateways must send pre-computed `prefix_hash`/`block_hashes` — otherwise -`LookupRoute` silently misses even with events flowing (see the "Block-size alignment" -note in [`docs/design/cachebackend-api.md`](../../../design/cachebackend-api.md)). This -managed wiring is exercised by the **controller/webhook envtests** — the SGLang -pod-injection, reserved-override, and admission tests — not reproduced step-by-step -here. (The install-smoke gate additionally admits the -`config/samples/cachebackend-sglang.yaml` shape against a real-cluster webhook via its -all-samples backstop, though it does not yet drive the SGLang reconcile or the full -inject→index→`LookupRoute` flow.) - -## Validate the event wire WITHOUT a GPU +```bash +kubectl -n cache-substrate get cachebackend sglang-lmcache-llama-8b -o yaml +kubectl -n cache-substrate get pod -l app=sglang-lmcache-llama-8b \ + -o jsonpath='{.items[0].spec.initContainers[*].name}{"\n"}' +kubectl -n cache-substrate logs deploy/sglang-lmcache-llama-8b \ + -c lmcache-mp-server --tail=100 +``` -Two complementary off-GPU checks, with an important scope distinction: +Expected operator surfaces include the `inferencecache.io/injected-by` +annotation, `ConnectorReady=True`, and `RemoteStorageReady=True` after Redis is +reachable. Redis loss should change the remote-storage condition without +rolling the engine pod. -1. **The shared decode + token-redaction path** (Python, no image/cluster). The - `kv_events_synthetic_publisher.py` emits the **2-field** `[ts, events]` EventBatch - envelope (vLLM's shape), so it exercises the decode + token-redaction logic common - to both engines — not SGLang's exact envelope: +## Functional check - ```bash - pip install -r ../../scripts/requirements.txt - python ../../scripts/kv_events_synthetic_publisher.py --bind 'tcp://*:5557' & - pub=$!; trap 'kill "$pub" 2>/dev/null' EXIT # background publisher; freed on exit even if a step below fails - python ../../scripts/kv_events_subscriber.py --endpoint tcp://localhost:5557 --max 4 - python ../../scripts/test_kv_events.py # asserts token_ids never surfaces; token_count kept - kill "$pub" 2>/dev/null; trap - EXIT - ``` +Port-forward the engine API and send the same long prefix twice, clearing only +the engine-local cache between the store and retrieve steps if you are proving +LMCache reuse. Keep the request model equal to the engine model and +`CacheBackend.spec.observation.modelID`. -2. **SGLang's exact wire shape** (Go, run from the repo root). SGLang's real envelope - is the **3-tuple** `[ts, events, attn_dp_rank]` with a 6-field `BlockStored`; that - shape — and that the subscriber's decoder tolerates the trailing `attn_dp_rank` and - tags reports `hash_scheme=sglang` — is asserted by the Go fixture the shipped - subscriber actually uses (from the repo root): - `go test ./internal/subscriber/ -run SGLang` - (`TestDecodeSGLangEventBatch` + `TestReporterTagsSGLangScheme`). The Python synthetic - path above does **not** cover the 3-tuple; rely on the Go test for the - SGLang-specific envelope. +```bash +kubectl -n cache-substrate port-forward \ + svc/sglang-lmcache-llama-8b 30000:30000 -## LMCache MP mode (how the offload works) +curl -sS http://127.0.0.1:30000/health +``` -SGLang drives LMCache in **MP (multiprocess) mode**, not the `lm://` remote-server -model the vLLM path uses. It ignores the `LMCACHE_*` env and reads config only from the -`--lmcache-config-file` flag, which points at the file the **MP worker** writes -(`mp_host: 127.0.0.1`, `mp_port: 5555`). The worker runs the LMCache MP server on -loopback, holds the L1 in `/dev/shm`, and offloads its shared tier to the Redis L2 over -the `resp` `--l2-adapter` (`lm://` is not a valid MP `--l2-adapter` type). The engine -attaches to the local worker over CUDA-IPC + shared memory; because they speak the MP -wire to each other, the worker defaults to the **same image** as the engine (same -lmcache version) — `spec.lmCache.workerImage` overrides it, at which point keeping the -two lmcache versions aligned is yours. Authoritative design + GPU-validation evidence: -[`docs/design/sglang-lmcache-mp-mode.md`](../../../design/sglang-lmcache-mp-mode.md); -key/field reference: [`docs/design/cachebackend-api.md`](../../../design/cachebackend-api.md) -(SGLang engine support). +The authoritative Phase 3 live evidence is recorded in +[`../../VERSIONS.md`](../../VERSIONS.md) and the +[migration roadmap](../../../design/lmcache-multiprocess-migration-roadmap.md). +It validated the controller-rendered SGLang TP=1 path, including host-only +retrieve, bounded L1 eviction, real events/status, and Redis retrieval by a +replacement engine pod. This hand manifest has not independently completed the +same GPU run with its placeholder replaced. -## Teardown +## Known limits -```bash -kind delete cluster --name inference-cache-substrate -``` +SGLang TP>1, multi-node execution, MLA, directional producer/consumer roles, +and MP-server restart/re-registration are outside this phase. Only +`spec.integration.role: ReadWrite` is supported. diff --git a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml index f4234526..24cfb452 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml +++ b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml @@ -2,49 +2,33 @@ # # SPDX-License-Identifier: Apache-2.0 -# SGLang + LMCache (MP mode) reference deployment — the second-engine sibling of -# the top-level vLLM + LMCache reference (manifests/deployment.yaml). +# SGLang + typed LMCache multiprocess reference deployment. The CacheBackend +# selects the engine pod and the operator injects the native lmcache-mp-server +# sidecar, MP config, and SGLang connector arguments. Redis is an explicit shared +# tier choice; no legacy LMCacheServer or Mooncake topology is inferred as Redis. # -# SGLang drives LMCache in MULTIPROCESS (MP) mode, so this stack differs from the -# vLLM reference on BOTH halves of the data plane: -# - the shared tier is a Redis L2 store (below), NOT a standalone lm:// server -# (lm:// is not a valid MP --l2-adapter type), and -# - the engine attaches to a node-local MP-worker sidecar over 127.0.0.1 -# (mp_host/mp_port) and reads its config from --lmcache-config-file; the old -# lm:// LMCACHE_REMOTE_URL env is gone (MP mode ignores it). -# This mirrors what the (sglang, LMCache) runtime adapter auto-injects on a matched -# engine pod (internal/adapters/builtin/runtime/sglang_lmcache.go + -# internal/adapters/builtin/runtime/sglang_lmcache_wire.go + -# internal/adapters/builtin/storage/redis.go), which is GPU-validated end to end. -# The engine image / --model-path / -# resources / --kv-events-config remain operator-owned scaffolding the adapter -# assumes is already present, so this is NOT byte-for-byte adapter output. +# The engine image, model, GPU resources, and KV-event publisher remain inference- +# system-owned. The engine image must contain a connector/package compatible with +# the injected LMCache server. Normal engine startup is the authoritative check. # -# VALIDATION STATUS: this hand manifest is derived from the GPU-validated adapter -# render (store->flush->retrieve reuses KV, via the controller-rendered managed -# path, #149) and is structurally checked (kubectl apply --dry-run=client). It has NOT been -# independently re-run end-to-end on a GPU in this exact shape — before treating it -# as a golden reference, a GPU run should confirm: engine + MP-worker startup, the -# config-file handoff, CUDA-IPC attachment, and a Redis store/retrieve (README step 4). -# Pins: docs/reference-stack/VERSIONS.md. +# VALIDATION STATUS: the typed SGLang renderer was GPU-validated through the +# controller path. This exact reference manifest has not independently completed a +# GPU run; see this directory's README and docs/reference-stack/VERSIONS.md. # # >> You need an NVIDIA GPU. << SGLang loads weights on CUDA and the LMCache MP -# worker moves KV over CUDA-IPC. There is no first-class CPU fallback here, so this +# server moves KV over CUDA-IPC. There is no first-class CPU fallback here, so this # manifest is GPU-only. SGLang's exact KV-event wire (the 3-tuple EventBatch # envelope) is covered off-GPU by `go test ./internal/subscriber/ -run SGLang`; the # Python scripts/ tooling exercises the shared decode + token-redaction (it models # vLLM's 2-tuple envelope). See this directory's README. # -# KUBERNETES >= 1.29 REQUIRED: the MP worker is a native sidecar (an initContainers +# KUBERNETES >= 1.29 REQUIRED: the MP server is a native sidecar (an initContainers # entry with restartPolicy: Always), which older apiservers do not understand. # -# IMAGE PINNING (see VERSIONS.md): the SGLang engine image ships as a non-applyable -# placeholder digest (the all-zero @sha256: below won't pull) — the base sglang -# image does NOT bundle the lmcache client, so you must build a DERIVED image -# (`pip install lmcache==0.5.1` onto the GPU-validated cu13 base) and substitute its -# real digest. The MP worker defaults to the SAME image (engine <-> worker must run -# the same lmcache version — they speak the MP wire). Redis is a normal pullable -# tag; digest-pin it for production per VERSIONS.md. +# IMAGE PINNING (see VERSIONS.md): the engine image is a deliberately non-pullable +# placeholder because this repository does not own inference engine images. Replace +# it with a digest-pinned compatible image. The CacheBackend pins the independently +# owned standalone server image. Digest-pin Redis for production as well. apiVersion: apps/v1 kind: Deployment metadata: @@ -54,8 +38,9 @@ metadata: app: redis-l2 spec: # SINGLETON: the Redis L2 holds no replicated state, so >1 replica would shard KV - # across independent keyspaces and silently partition the tier. Keep replicas: 1 - # (the managed path's admission enforces this for a (sglang, LMCache) backend). + # across independent keyspaces and silently partition the tier. Keep replicas: 1. + # This manifest owns Redis directly and binds it as External storage, so the + # CacheBackend controller does not manage or validate this Deployment. replicas: 1 selector: matchLabels: @@ -79,8 +64,8 @@ spec: # (replaces CMD, keeps ENTRYPOINT). --save ""/--appendonly no: ephemeral # cache tier, no persistence/PVC (a restart starts cold — soft state). # --maxmemory + allkeys-lru bound the dataset so write load can't OOM the - # cgroup; the managed path derives ~80% of the memory LIMIT (below: the - # byte-exact value for an 8Gi limit, 8589934592 - 8589934592/5 = + # cgroup. This mirrors the managed renderer's ~80% derivation (below: + # the byte-exact value for an 8Gi limit, 8589934592 - 8589934592/5 = # 6871947674 ≈ 6.4Gi). --protected-mode no: the worker dials over the # ClusterIP (a non-loopback client). args: @@ -92,7 +77,7 @@ spec: - "--protected-mode" - "no" - "--maxmemory" - - "6871947674" # bytes = ~80% of the 8Gi limit below (matches the managed render) + - "6871947674" # bytes = ~80% of the 8Gi limit below - "--maxmemory-policy" - "allkeys-lru" ports: @@ -122,6 +107,42 @@ spec: ports: - { name: redis, port: 6379, targetPort: redis } --- +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-lmcache-llama-8b + namespace: cache-substrate +spec: + runtime: SGLang + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + app: sglang-lmcache-llama-8b + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi + remoteStorage: + provider: Redis + ownership: External + endpoint: redis-l2:6379 + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -143,71 +164,6 @@ spec: app: sglang-lmcache-llama-8b app.kubernetes.io/name: sglang spec: - # The MP worker is a NATIVE SIDECAR: an initContainers entry with - # restartPolicy: Always, so it starts (and its startupProbe gates the engine) - # before the engine container, and keeps running for the pod's life. Requires - # Kubernetes >= 1.29. - initContainers: - - name: lmcache-mp-worker - restartPolicy: Always - # Defaults to the ENGINE image so the worker and engine run the same - # lmcache version (they speak the MP wire). Same non-applyable placeholder - # digest as the engine — substitute your derived image (VERSIONS.md). - image: example.invalid/sglang-lmcache@sha256:0000000000000000000000000000000000000000000000000000000000000000 - imagePullPolicy: IfNotPresent - # Write the engine's MP config file (mp_host/mp_port), then exec the - # LMCache MP server on loopback, offloading to the Redis L2 via `resp`. - # This is the shape the adapter renders; keep chunk_size/mp_port/l1-size-gb - # in sync with the config the engine reads (below) and the shm sizeLimit. - command: ["sh", "-c"] - args: - - | - set -e - printf 'chunk_size: 256\nmp_host: "127.0.0.1"\nmp_port: 5555\n' > /etc/lmcache/config.yaml - exec python3 -m lmcache.v1.multiprocess.server \ - --host 127.0.0.1 --port 5555 \ - --chunk-size 256 --l1-size-gb 4 --eviction-policy LRU \ - --l2-adapter '{"type":"resp","host":"redis-l2","port":6379}' - env: - # The GPU-less worker must SEE the engine's GPU to CUDA-IPC its KV. It - # holds no nvidia.com/gpu request (consumes none of the node's - # allocatable), but the visibility is real — on a shared node it can see - # every GPU. GPU-VALIDATED as required: revoking it kills the worker - # ("Device UUID ... not found") and the engine never readies. See the - # GPU-visibility note in docs/design/cachebackend-api.md. - - { name: NVIDIA_VISIBLE_DEVICES, value: "all" } - volumeMounts: - - { name: lmcache-config, mountPath: /etc/lmcache } - - { name: shm, mountPath: /dev/shm } # shared L1 tmpfs with the engine - # The MP server binds mp_port on loopback, which a pod-IP probe can't - # reach — exec a loopback check so the engine only starts once the worker - # is listening. Gating the engine on the worker is deliberate: SGLang has - # no cacheless fallback while --enable-lmcache is on (see "Fail-open - # semantics" in docs/design/sglang-lmcache-mp-mode.md). - startupProbe: - exec: - command: - - python3 - - -c - - "import socket; socket.create_connection(('127.0.0.1',5555),1)" - periodSeconds: 3 - failureThreshold: 40 - # The L1 lives in the memory-backed /dev/shm tmpfs, charged to this - # container's cgroup, so it MUST carry a matching memory request+limit - # (l1-size-gb + ~1Gi headroom) or the L1 is invisible to the scheduler. - resources: - requests: { memory: 5Gi } - limits: { memory: 5Gi } - # Restricted-compatible: this mutation lands before Pod Security admission, - # so the worker must carry the container-only Restricted requirements - # itself or it would get the whole engine pod rejected in a restricted - # namespace. No added capabilities (KV moves over CUDA-IPC + /dev/shm). - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - seccompProfile: - type: RuntimeDefault containers: # Container name MUST be `sglang` so the adapter's InjectEngineConfig / # engineOverrides target the right container (EngineContainerName()). @@ -229,18 +185,8 @@ spec: - "--model-path=meta-llama/Meta-Llama-3-8B-Instruct" - "--host=0.0.0.0" - "--port=30000" - # Prometheus /metrics on :30000 (store_true flag). SGLang defaults it - # OFF (unlike vLLM), so without it the kvevent-subscriber's stats path - # has nothing to scrape and load-aware routing goes dark. The managed - # path injects this automatically; the reference sets it explicitly. - - "--enable-metrics" - # SGLang's LMCache connector (store_true flag — no value). Replaces - # vLLM's --kv-transfer-config. - - "--enable-lmcache" - # Points the engine at the MP config file the worker writes (mp_host/ - # mp_port). MP mode aborts at startup WITHOUT this flag. - - "--lmcache-config-file" - - "/etc/lmcache/config.yaml" + # The CacheBackend webhook injects the SGLang LMCache MP arguments and + # config path. Keep only inference-system-owned arguments here. # KV-cache event publisher — SGLang's ZmqEventPublisher, same event # structs as vLLM (3-tuple batch envelope). Consumer is the Go # kvevent-subscriber (managed path). Binds tcp://*:5557 — the operator @@ -252,15 +198,6 @@ spec: - "--kv-events-config" - '{"publisher":"zmq","endpoint":"tcp://*:5557","buffer_steps":10000,"topic":"kv-events"}' env: - # Gates SGLang's experimental LMCache integration — required for - # --enable-lmcache to engage the connector. - - { name: LMCACHE_USE_EXPERIMENTAL, value: "True" } - # Mirror of spec.integration.failOpen (defaults true) — the adapter - # injects this as a reserved env so the engine layer can honor the - # fail-open contract. (The old lm:// env — LMCACHE_REMOTE_URL / SERDE / - # CHUNK_SIZE / LOCAL_CPU — is intentionally absent: MP mode ignores it, - # and the MP worker's tunables live in its config file / CLI above.) - - { name: INFERENCECACHE_FAIL_OPEN, value: "true" } # Gated model — needs a Hugging Face token (see README step 2). - name: HF_TOKEN valueFrom: @@ -278,20 +215,16 @@ spec: nvidia.com/gpu: "1" volumeMounts: - { name: cache-home, mountPath: /root/.cache/huggingface } - - { name: shm, mountPath: /dev/shm } # shared L1 tmpfs with the MP worker - - { name: lmcache-config, mountPath: /etc/lmcache } # reads the worker's config.yaml + - { name: shm, mountPath: /dev/shm } # shared with the injected MP server volumes: - name: cache-home emptyDir: {} - # The MP L1 tier lives here; shared by the engine and the worker. Sized to - # cover the engine's own /dev/shm use PLUS the L1 budget (l1-size-gb=4). + # The MP L1 tier lives here; shared by the engine and injected server. Sized + # to cover the engine's own /dev/shm use plus the 4Gi typed L1 budget. - name: shm emptyDir: medium: Memory sizeLimit: 8Gi - # Carries the MP config file the worker writes and the engine reads. - - name: lmcache-config - emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/docs/reference-stack/scripts/canary_c2_reconcile.sh b/docs/reference-stack/scripts/canary_c2_reconcile.sh index 3c3c20b9..620428dd 100755 --- a/docs/reference-stack/scripts/canary_c2_reconcile.sh +++ b/docs/reference-stack/scripts/canary_c2_reconcile.sh @@ -4,6 +4,10 @@ # # SPDX-License-Identifier: Apache-2.0 +# LEGACY IP COMPATIBILITY ONLY. Retained until Phase 7; do not use this script +# as a production deployment reference. It intentionally exercises +# remoteStorage.provider=LMCacheServer and the lm:// data plane. +# # Canary for the C2 CacheBackend reconciler. Proves the controller stands up a # healthy, serving backend from a CR on a GPU-free cluster (kind): # @@ -23,7 +27,7 @@ # can't cover. The managed standalone server uses CPU storage and does not need # an inference engine or GPU for this controller lifecycle check. # -# On-demand canary (NOT a per-PR gate): needs Docker + kind + kubectl, pulls the +# Manual canary (NOT a per-PR gate): needs Docker + kind + kubectl, pulls the # standalone LMCache server image. See docs/reference-stack/VERSIONS.md. # # Usage: docs/reference-stack/scripts/canary_c2_reconcile.sh diff --git a/docs/reference-stack/scripts/canary_c6_engine_wiring.sh b/docs/reference-stack/scripts/canary_c6_engine_wiring.sh index b4413650..73a5d7ee 100755 --- a/docs/reference-stack/scripts/canary_c6_engine_wiring.sh +++ b/docs/reference-stack/scripts/canary_c6_engine_wiring.sh @@ -4,6 +4,10 @@ # # SPDX-License-Identifier: Apache-2.0 +# LEGACY IP COMPATIBILITY ONLY. Retained until Phase 7; do not use this script +# as a production deployment reference. It intentionally exercises +# LMCacheConnectorV1, LMCACHE_REMOTE_URL, and LMCacheServer. +# # CPU canary for the C6 mutating Pod webhook + cross-pod cache reuse. # # Proves the engine-wiring webhook injects the LMCache connector env onto @@ -25,7 +29,7 @@ # is ~5 GiB per pod, so the Docker VM needs ~12 GiB RAM (see # docs/reference-stack/VERSIONS.md for the documented memory floor). # Pulls the multi-GB vLLM image. This is NOT a per-PR gate; it runs on a -# schedule and on manual dispatch. +# manual dispatch only. # # Usage: docs/reference-stack/scripts/canary_c6_engine_wiring.sh # Tunables: IMAGE, MODEL, KIND_CLUSTER, NAMESPACE, READY_TIMEOUT, diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index e3c4169a..cbafdbf9 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -4,6 +4,11 @@ # # SPDX-License-Identifier: Apache-2.0 +# Phase-5 classification: typed PodLocal MP checks in this script are current. +# Every LMCacheServer, LMCacheConnectorV1/LMCACHE_REMOTE_URL, lm://, or Mooncake +# case is an intentional legacy-IP compatibility assertion retained only until +# Phase 7; none is a production fixture or recommended deployment path. + # Per-PR install smoke for `kubectl apply -k config/default`. # # Builds controller + server images at a deterministic tag, loads them into a @@ -67,7 +72,7 @@ # controller carries `--lmcache-server-image`, the smoke rewrites it to a # locally built stand-in, and a CacheBackend with no CR-level image renders # that configured image into its owned Deployment. -# 8b. Provider resource fallback: the paired sample leaves +# 8b. Legacy-IP compatibility: an inline fixture leaves # remoteStorage.lmCacheServer.resources unset, while the provider renderer # gives the cache-server container a 4Gi request / 8Gi limit. The smoke # asserts the CR remains unchanged and the rendered pod is still bounded @@ -86,8 +91,7 @@ # is persisted through the same live webhook and carries the dedicated # LMCacheMPConnector module path, loopback MP endpoint, deterministic hash # seed, hybrid-manager guard, and no legacy lm:// environment. -# 9. Canonical External ownership end-to-end: applying the committed -# config/samples/cachebackend-external.yaml drives the CacheBackend +# 9. Legacy-IP External compatibility: an inline fixture drives the CacheBackend # mutating webhook default (spec.replicas=1), renders NO # Deployment/Service in its namespace, status.endpoint mirrors # spec.remoteStorage.endpoint, observedGeneration is set, the CR goes @@ -312,8 +316,8 @@ KERNEL_CHECK_POD_TIMEOUT="${KERNEL_CHECK_POD_TIMEOUT:-120}" # and publish EngineKernelsHealthy=False. One reconcile cycle + poll buffer. KERNEL_CHECK_COND_TIMEOUT="${KERNEL_CHECK_COND_TIMEOUT:-60}" -# Sample-smoke tunables — apply config/samples/cachebackend-with-engine.yaml, -# assert the operator-facing signals, exercise the RequeueAfter drift case. +# Legacy-IP paired-binding smoke tunables. The current paired sample contributes +# only the engine scaffold; the legacy CacheBackend fixture is inline. # # Default namespace is dedicated to this smoke so re-runs against an existing # cluster (KEEP_CLUSTER=1) don't mutate or delete a developer's own resources @@ -1587,8 +1591,14 @@ log "routingFloorScore=0.1 restored: same 64-token match flipped back to PREFIX_ kubectl delete namespace "$POLICY_SMOKE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true -# --- paired-sample smoke --------------------------------------------------- -# Applies config/samples/cachebackend-with-engine.yaml and asserts the +# --- legacy IP paired-binding compatibility smoke -------------------------- +# INTENTIONAL LEGACY FIXTURE: this section exercises the LMCacheServer/IP +# implementation retained until Phase 7. It is not a production reference. +# The current config/samples/cachebackend-with-engine.yaml supplies only the +# inference-system-owned engine scaffold; the legacy CacheBackend is written +# inline below. The current typed MP path is exercised in the canonical section. +# +# Asserts the # CacheBackend ↔ engine-pod binding's operator-facing signals materialize # end-to-end: # - status.matchedEnginePods → 1 @@ -1617,7 +1627,7 @@ log "creating sample namespace $SAMPLE_NS" kubectl create namespace "$SAMPLE_NS" --dry-run=client -o yaml \ | kubectl apply -f - >/dev/null -log "splitting paired sample into CacheBackend doc and engine Deployment doc" +log "splitting the current paired sample to reuse its engine scaffold" sample_file="config/samples/cachebackend-with-engine.yaml" # Place the split files under the trapped $tmpdir so an early failure # between split and apply (or a SIGINT mid-run) does not leak temp files @@ -1634,6 +1644,33 @@ awk -v cb="$sample_tmp_cb" -v engine="$sample_tmp_engine" ' sep { print > engine } ' "$sample_file" +# Replace the typed MP CacheBackend document with an explicitly isolated legacy +# fixture. Keeping it inline prevents a repository-owned sample from presenting +# LMCacheServer as a supported deployment choice. +cat >"$sample_tmp_cb" <<'EOF' +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: qwen-demo-cache +spec: + runtime: VLLM + type: LMCache + deploymentKind: Deployment + replicas: 1 + integration: + role: ReadWrite + engineSelector: + matchLabels: + app: qwen-demo + observation: + modelID: Qwen/Qwen2.5-0.5B-Instruct + remoteStorage: + provider: LMCacheServer + ownership: Managed + lmCacheServer: + image: lmcache/standalone:v0.4.7 +EOF + # Patch the engine container's image to the lightweight stand-in. The # Deployment's metadata stays untouched (still qwen-engine, still # labeled app=qwen-demo), so the binding label flow is exercised exactly @@ -1644,7 +1681,7 @@ rm -f "${sample_tmp_engine}.bak" build_sample_cache_server_image if ! grep -q '^ image: lmcache/standalone:v0.4.7$' "$sample_tmp_cb"; then - fail "fixture: cachebackend-with-engine.yaml no longer carries the pinned canonical remoteStorage.lmCacheServer.image" + fail "legacy inline fixture no longer carries remoteStorage.lmCacheServer.image" fi sed -i.bak '/^ image: lmcache\/standalone:v0.4.7$/d' "$sample_tmp_cb" rm -f "${sample_tmp_cb}.bak" @@ -2445,8 +2482,12 @@ log "typed vLLM PodLocal Pod persisted with the dedicated external MP connector kubectl delete namespace "$CANONICAL_SMOKE_NS" \ --wait=false --ignore-not-found=true >/dev/null 2>&1 || true -# --- External CacheBackend end-to-end --------------------------------------- -# Exercises the committed External passthrough sample on the running cluster: +# --- legacy IP External compatibility -------------------------------------- +# INTENTIONAL LEGACY FIXTURE: this section exercises External LMCacheServer/IP +# validation and injection retained until Phase 7. It is not a production +# reference. Current External Redis is covered by typed schema/sample checks. +# +# Exercises External passthrough on the running cluster: # the mutating webhook should stamp spec.replicas, the reconciler should NOT # render a Deployment/Service, status.endpoint should mirror # spec.remoteStorage.endpoint, @@ -2458,11 +2499,28 @@ kubectl delete namespace "$CANONICAL_SMOKE_NS" \ log "exercising External CacheBackend end-to-end in namespace $EXT_SMOKE_NS" kubectl create namespace "$EXT_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -# Apply the committed External CR sample. The sample intentionally omits -# spec.replicas so the smoke drives the mutating webhook defaulter instead of -# only proving the CRD accepts already-defaulted YAML. -kubectl -n "$EXT_SMOKE_NS" apply -f config/samples/cachebackend-external.yaml >/dev/null \ - || fail "kubectl apply config/samples/cachebackend-external.yaml failed" +# The inline fixture intentionally omits spec.replicas so the smoke drives the +# mutating webhook defaulter instead of only proving the CRD accepts already- +# defaulted YAML. +cat </dev/null \ + || fail "kubectl apply legacy External LMCacheServer fixture failed" +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: $EXT_SMOKE_CB_NAME +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + app.kubernetes.io/name: vllm + remoteStorage: + provider: LMCacheServer + ownership: External + endpoint: my-cache.example.com:8200 +EOF defaulted_replicas="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" @@ -4048,9 +4106,12 @@ if ! grep -q "\"exitCode\": $doctor_rc" "$LOG_DIR/doctor.json"; then fi log "inferencecache doctor ran against the live install (exit $doctor_rc; JSON envelope + CB finding present)" -# --- managed Mooncake backend smoke ---------------------------------------- -# Canonical CacheBackend{runtime: VLLM, type: LMCache, -# remoteStorage.provider: Mooncake} is an operator-facing surface, so it needs +# --- legacy IP managed Mooncake compatibility smoke ------------------------ +# INTENTIONAL LEGACY FIXTURE: Mooncake-through-LMCache/IP remains implemented +# until Phase 7, but is not a current production path or sample. This section +# keeps compatibility coverage without presenting it as a recommended backend. +# CacheBackend{runtime: VLLM, type: LMCache, +# remoteStorage.provider: Mooncake} is still an operator-facing alpha surface, so it needs # a real-install assertion, not just unit/envtest. The kvcacheai/mooncake image # is intentionally NOT pulled here (heavy, and its entrypoint/ports are pending # reference-stack validation); instead a busybox stand-in named `mooncake_master` @@ -4087,10 +4148,33 @@ log "creating namespace $MOONCAKE_SMOKE_NS" kubectl create namespace "$MOONCAKE_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null mc_cb_tmp="$(mktemp "$tmpdir/mooncake-cb.XXXXXX")" -cp config/samples/cachebackend-mooncake.yaml "$mc_cb_tmp" +cat >"$mc_cb_tmp" <<'EOF' +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: cachebackend-mooncake +spec: + runtime: VLLM + type: LMCache + deploymentKind: Deployment + replicas: 1 + integration: + role: ReadWrite + engineHostNetwork: true + engineSelector: + matchLabels: + app.kubernetes.io/name: vllm + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct + remoteStorage: + provider: Mooncake + ownership: Managed + mooncake: + image: docker.io/kvcacheai/mooncake:0.3.11.post1 +EOF mc_escaped_image="$(printf '%s' "$MOONCAKE_MASTER_IMAGE" | sed 's/[&|\\]/\\&/g')" if ! grep -q '^ image: docker.io/kvcacheai/mooncake:0.3.11.post1$' "$mc_cb_tmp"; then - fail "fixture: cachebackend-mooncake.yaml no longer carries the pinned canonical remoteStorage.mooncake.image" + fail "legacy inline Mooncake fixture no longer carries remoteStorage.mooncake.image" fi sed -i.bak "s|^ image: docker.io/kvcacheai/mooncake:0.3.11.post1$| image: $mc_escaped_image|g" "$mc_cb_tmp" rm -f "${mc_cb_tmp}.bak" @@ -4114,7 +4198,7 @@ log "applying Mooncake CacheBackend" # re-indented, the sed below silently no-ops and the first assertion then fails # with "did not warn" — blaming the webhook for a broken test fixture. if ! grep -q '^ engineHostNetwork: true$' "$mc_cb_tmp"; then - fail "fixture: cachebackend-mooncake.yaml no longer carries 'engineHostNetwork: true' at the expected indent; the no-opt-in copy would be a no-op" + fail "legacy inline Mooncake fixture no longer carries 'engineHostNetwork: true' at the expected indent; the no-opt-in copy would be a no-op" fi mc_nooptin_tmp="$(mktemp "$tmpdir/mooncake-cb-nooptin.XXXXXX")" sed 's/^ engineHostNetwork: true$//' "$mc_cb_tmp" >"$mc_nooptin_tmp" @@ -4136,15 +4220,15 @@ case "$mc_apply_out" in log "Mooncake with the opt-in applies without the engine-hostNetwork warning" ;; esac -# Pin the fixture to the canonical hierarchy. +# Pin the fixture to the retained legacy hierarchy. mc_runtime="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.runtime}')" mc_type="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.type}')" mc_provider="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.remoteStorage.provider}')" if [ "$mc_runtime" != "VLLM" ] || [ "$mc_type" != "LMCache" ] || [ "$mc_provider" != "Mooncake" ]; then kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake smoke fixture is not canonical: runtime=$mc_runtime type=$mc_type remoteStorage.provider=$mc_provider" + fail "Mooncake compatibility fixture changed shape: runtime=$mc_runtime type=$mc_type remoteStorage.provider=$mc_provider" fi -log "canonical Mooncake CacheBackend admitted (runtime=VLLM, type=LMCache, provider=Mooncake)" +log "legacy Mooncake CacheBackend admitted (runtime=VLLM, type=LMCache, provider=Mooncake)" # Reuses SAMPLE_ENDPOINT_TIMEOUT deliberately: the reconcile-to-status.endpoint # latency is a per-managed-backend property (the reconciler publishes it from diff --git a/site/content/en/docs/concepts/cachebackend.md b/site/content/en/docs/concepts/cachebackend.md index 3ce53592..194d4047 100644 --- a/site/content/en/docs/concepts/cachebackend.md +++ b/site/content/en/docs/concepts/cachebackend.md @@ -3,27 +3,17 @@ title: "CacheBackend" linkTitle: "CacheBackend" weight: 2 description: > - The primary resource: bind engine pods to a KV-cache backend and make their KV cache - reusable across requests. + Bind inference-engine Pods to a typed cache data plane and expose cache-aware routing state. --- ## What is a CacheBackend? -A `CacheBackend` is the primary CRD an operator writes. It describes a shared KV-cache -backend and the engine-integration policy that uses it. Applying one: +A namespaced `CacheBackend` selects an inference runtime, its engine-side cache +integration, and an optional remote L3. The inference system owns the engine +Deployment and image. CacheBackend injects the selected connector and the cache +components it owns into matching Pods. -1. **Provisions** a managed cache-server workload (for backend types that need one) and a - `ClusterIP` Service. -2. **Binds** to inference-engine pods by label (`spec.engineSelector`). The mutating Pod - webhook injects the KV-connector configuration into matching pods. It also injects the - observation sidecar when the controller has `--kvevent-subscriber-image` configured and - `spec.observation.modelID` is set. -3. **Makes the engine's KV cache reusable** — offloaded to the backend (tier 2) and, when - subscriber reporting is enabled, surfaced to routing (tier 1) so a warm prefix skips - prefill. - -`CacheBackend` is namespaced. Group `inferencecache.io`, version `v1alpha1`, short name -`cb`. +Current LMCache offload uses typed PodLocal multiprocess mode: ```yaml apiVersion: inferencecache.io/v1alpha1 @@ -35,163 +25,81 @@ spec: runtime: VLLM type: LMCache integration: - mode: Offload role: ReadWrite engineSelector: matchLabels: app: llama3-vllm + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: modelID: meta-llama/Llama-3.1-8B-Instruct - remoteStorage: - provider: LMCacheServer - ownership: Managed - lmCacheServer: - resources: - requests: - memory: 4Gi - limits: - memory: 8Gi ``` -## Backend types (`spec.type`) - -`spec.type` is the engine-side cache implementation. It is a CRD enum and -defaults to `LMCache`. +Omitting `remoteStorage` intentionally selects host-only MP. L1 is per engine +Pod; add an explicit managed or external Redis L3 when cross-Pod sharing is +required. -| Type | What it is | -|---|---| -| **`LMCache`** (default) | LMCache engine integration. `spec.remoteStorage` independently selects an optional remote provider. | -| **`SGLangHiCache`** | SGLang's native engine-local host cache; it accepts no remote-storage binding. | - -The runtime + cache-type **pair** selects the engine adapter. Remote provider -technology (`Redis`, `LMCacheServer`, or `Mooncake`) and lifecycle ownership -(`Managed` or `External`) are selected under `spec.remoteStorage`. Admission -rejects unsupported combinations. - -{{% alert title="LMCache durability" color="info" %}} -The `lm://` LMCache server is **in-memory only.** Durability is a *backend choice*, not a -generic volume knob — there is no per-`CacheBackend` PVC field. If you need a durable or -shared store, use `remoteStorage.provider: Mooncake`. +{{% alert title="Runtime-owned engine image" color="info" %}} +CacheBackend never rewrites the engine image. Normal engine initialization is +the authoritative check that the image contains a compatible LMCache client. +PodLocal native sidecars require Kubernetes 1.29 or newer. {{% /alert %}} -### Mooncake needs host networking - -Mooncake is a peer-to-peer transfer-engine mesh: its master returns a *directory pointer* -("block X is on node B, port P"), and the engine then connects directly to that node's real -IP on a dynamically chosen port to move the KV bytes. A single ClusterIP cannot route that. - -Consequently, a Mooncake backend renders the master with `hostNetwork: true` behind a -headless Service, and the **engine pods also need host networking** — opt in with -`spec.integration.engineHostNetwork: true`. The namespace must permit `hostNetwork`, the -backend becomes a singleton (`replicas > 1` and autoscaling are rejected), and one engine -per node per port applies. TCP transport is sufficient for correctness; RDMA/RoCE only -affects bandwidth. - -## Engine integration (`spec.integration`) - -| Field | Values | Meaning | -|---|---|---| -| `mode` | `Offload` (default), `EventsOnly` | `Offload` = routing + tier-2 offload + a provisioned server. `EventsOnly` = routing only, no server, no KV connector. | -| `role` | `ReadOnly`, `WriteOnly`, `ReadWrite` (default) | Maps to the LMCache `kv_role` (`kv_consumer` / `kv_producer` / `kv_both`). | -| `failOpen` | `true` (default) | The engine falls back to local prefill when the cache is unreachable. `false` fails closed (and emits a Warning Event). | -| `engineOverrides` | — | Fine-grained control over injected args/env (see below). | -| `engineHostNetwork` | `false` (default) | Opt-in host networking for Mooncake engine pods. | - -### Events-only mode - -`mode: EventsOnly` provisions **no** cache server — routing tier only. It exists for -**hybrid-attention models** (for example gated-DeltaNet, Mamba/Jamba, Falcon-H, -Granite-hybrid families) that cannot take a vLLM KV connector, because vLLM disables its -hybrid KV-cache manager when any connector loads. The subscriber sidecar is still injected -so routing hints stay live; evictions are forwarded as `PREFIX_EVICTED`. Events-only -requires `type: LMCache`, forbids `autoscaling`, and is incompatible with `External`. - -### Engine-injection overrides +## Cache types and remote storage -`spec.integration.engineOverrides` exposes four primitives that merge on top of the -adapter's canonical injection: - -- `args` — extra engine args to add. -- `suppressArgs` — canonical args to remove. -- `env` — extra environment variables to add. -- `suppressEnv` — canonical env vars to remove. - -Each adapter declares **reserved** args and env that carry correctness guarantees. -Overriding a reserved value is **hard-rejected at admission**, not merely warned — a warning -would be ignored and the engine would crash later with no breadcrumb. See -[Bind an engine]({{< relref "/docs/tasks/bind-an-engine/" >}}) for the reserved lists per runtime. - -## Engine binding (`spec.engineSelector`) - -`spec.engineSelector.matchLabels` is an equality selector over engine **pod** labels -(`matchExpressions` is not available in v1alpha1). Any pod whose labels match, in the -`CacheBackend`'s namespace, is a target for the mutating Pod webhook. `status.matchedEnginePods` -reports how many pods currently match. - -A pod carrying the annotation `inferencecache.io/skip-inject: "true"` is skipped entirely — -the all-or-nothing escape hatch. +| Type | Current behavior | +|---|---| +| `LMCache` | Typed PodLocal MP for vLLM or SGLang; optional Redis L3. | +| `SGLangHiCache` | SGLang's native engine-local host cache; no remote binding. | -The subscriber sets `replica_id = `. Treat that ID as opaque: engine Deployment -names do not need to equal the `CacheBackend` name. The controller attributes pods to a -backend through `spec.engineSelector` and the webhook's `inferencecache.io/injected-by` -metadata, not a pod-name prefix. +`remoteStorage` is optional L3 only. Redis may be `Managed` or `External`. +Legacy topology-less `LMCacheServer` and engine-side Mooncake shapes remain in +the alpha schema only for compatibility until migration Phase 7; they are not +current production profiles and are not automatically mapped to Redis. -## Readiness +## Engine integration -`CacheBackend.Ready` composes three gates, in order: +| Field | Meaning | +|---|---| +| `mode` | `Offload` by default; `EventsOnly` observes KV events without injecting a cache connector. | +| `role` | LMCache currently admits only `ReadWrite`; directional roles are future work. | +| `failOpen` | Defaults to `true`; remote L3 loss degrades independently from the required PodLocal connector. | +| `engineOverrides` | Amends non-reserved engine args/env. Use typed `lmCache` fields for MP configuration. | -1. **Managed-readiness baseline** — the provisioned workload is available (or scaled to - zero, or rolling). -2. **KV-event gate** — Ready stays `False` (`AwaitingFirstKVEvent`) until a *real* engine - pod has published at least one KV event for this backend. This proves the publisher - works, not just that the pod IP is reachable. After `firstEventTimeout` with no events it - reports `NoKVEventsObserved` / `Degraded`. Opt out per-CR with - `inferencecache.io/require-kv-events: "false"`. -3. **Functional-probe gate** — the controller drives a synthetic round-trip through the - server's `/probe` endpoint (ingest → routing → tier-2). `FunctionalProbeOK` appears only - after this clears. Opt out with `inferencecache.io/skip-functional-probe: "true"`. +The webhook binds Pods by `spec.engineSelector.matchLabels` at Pod CREATE. A +Pod can opt out with `inferencecache.io/skip-inject: "true"`. Recreate Pods +after changing binding or cache configuration. -`External` backends are exempt from the KV-event and probe gates — only endpoint acceptance -and Ready are checked. +## Readiness and status -## Status +Typed MP exposes connector and remote-storage health separately: -Selected `status` fields: +- `ConnectorReady` covers selected engines and their required PodLocal MP + servers; +- `RemoteStorageReady` is present only when a Redis L3 is configured; and +- `Ready` composes the implemented readiness and observation gates. -| Field | Meaning | -|---|---| -| `endpoint` | The resolved backend endpoint. | -| `matchedEnginePods` | Pointer — `nil` (not yet observed) vs a real count. | -| `failOpen` | The effective fail-open posture. | -| `firstKVEventObservedAt` | Write-once latch — the first time any KV event was seen. | -| `indexParticipation` | `prefixCount`, `lastEventAt`, `hitRate`, `t2HitRate` — this backend's slice of the index. | -| `conditions` | The authoritative health surface (see below). | -| `observedGeneration` | Standard reconcile bookkeeping. | - -Printer columns: `Type`, `Ready`, `Matched`, `Endpoint`, `Prefixes`, `LastEvent`, `Age`. - -### Conditions - -Conditions are the authoritative health surface (there is no single `Health` enum). An -Offload-managed backend can publish up to seven: `Ready`, `Degraded`, `Progressing`, -`FunctionalProbeOK`, `EngineKernelsHealthy`, `T2Degraded`, `EngineCompatibility`. -Events-only publishes three (`Ready`/`Degraded`/`Progressing`); `External` publishes -`Ready` + `Progressing`. - -Two advisory conditions worth calling out: - -- **`T2Degraded`** — sourced from `status.indexParticipation.t2HitRate`. `True/T2ZeroHitRate` - means the tier-2 offload was queried but served zero reloads — a silently-degraded tier. - It never gates `Ready`. (Because it is lifetime-cumulative, a *mid-life* regression is - caught by the `LMCacheT2NoHits` alert instead.) -- **`EngineCompatibility`** — `False/InjectedEngineCrashLooping` flags an engine that is - crash-looping after injection (often a hybrid-attention model that cannot take a - connector). The remedy is to switch that backend to events-only mode, **not** to - `skip-inject`. +`status.endpoint` is empty for host-only PodLocal MP and contains only a remote +L3 endpoint. It never publishes the loopback connector address. Other useful +fields include `matchedEnginePods`, `connector`, `remoteStorage`, +`indexParticipation`, `conditions`, and `observedGeneration`. ## Related pages -- [Bind an engine]({{< relref "/docs/tasks/bind-an-engine/" >}}) — the injection contract, reserved - args/env, and the skip annotation. -- [CachePolicy]({{< relref "/docs/concepts/cachepolicy/" >}}) — per-namespace lookup and eviction tuning. -- [CRD API reference]({{< relref "/docs/reference/crd-api/" >}}) — every field. +- [Bind an engine]({{< relref "/docs/tasks/bind-an-engine/" >}}) +- [CachePolicy]({{< relref "/docs/concepts/cachepolicy/" >}}) +- [CRD API reference]({{< relref "/docs/reference/crd-api/" >}}) diff --git a/site/content/en/docs/concepts/pdtopology.md b/site/content/en/docs/concepts/pdtopology.md index db4673cc..dec89888 100644 --- a/site/content/en/docs/concepts/pdtopology.md +++ b/site/content/en/docs/concepts/pdtopology.md @@ -67,4 +67,4 @@ defines the shape the full implementation will consume. ## Related pages - [The gRPC contract]({{< relref "/docs/concepts/grpc-contract/" >}}) — the `LookupPDRoute` RPC. -- [CacheBackend]({{< relref "/docs/concepts/cachebackend/" >}}) — Mooncake as a transfer backend. +- [CacheBackend]({{< relref "/docs/concepts/cachebackend/" >}}) — current cache integration profiles. diff --git a/site/content/en/docs/overview/_index.md b/site/content/en/docs/overview/_index.md index 759c4d64..df5b768f 100644 --- a/site/content/en/docs/overview/_index.md +++ b/site/content/en/docs/overview/_index.md @@ -11,7 +11,7 @@ control plane for LLM inference**. It makes routing and cache decisions *cache-a that requests land on replicas that already hold the prompt's KV cache warm — turning a prefix cache hit into lower time-to-first-token (TTFT) and lower cost. -It **orchestrates** the KV-cache technology you already run (LMCache, Mooncake) rather than +It **orchestrates** the KV-cache technology you already run (currently LMCache MP) rather than replacing it. inference-cache is **not** a new distributed cache, and it is **not** the data-plane gateway. It is the brain that decides *where a request should go*; the gateway follows the hint. @@ -40,9 +40,8 @@ for every gateway client, and no routing logic fragmented across clients. webhook injects the KV-connector configuration automatically. The observation sidecar is also injected when the controller's opt-in `--kvevent-subscriber-image` is configured. -- 🧩 **Pluggable KV-cache backends** — the default in-memory LMCache backend for the simple - path; Mooncake for a durable, shared, peer-to-peer store; or point at an `External` - endpoint you manage. +- 🧩 **Typed LMCache MP** — one PodLocal MP server per engine Pod, with optional + managed or external Redis L3 selected explicitly when cross-Pod sharing is required. - 🏠 **Multi-tenant by construction** — the cache-state index is keyed by `(tenant, model, hash_scheme, adapter_id, prefix_hash)`, so tenants' hints can never diff --git a/site/content/en/docs/reference/crd-api.md b/site/content/en/docs/reference/crd-api.md index 83c6e23d..92c687dc 100644 --- a/site/content/en/docs/reference/crd-api.md +++ b/site/content/en/docs/reference/crd-api.md @@ -10,7 +10,7 @@ All CRDs are in the API group **`inferencecache.io`**, version **`v1alpha1`**. | Kind | Scope | Short name | Reconciled? | Purpose | |---|---|---|---|---| -| `CacheBackend` | Namespaced | `cb` | Yes | Bind engine pods to a KV-cache backend; provision the managed cache server. | +| `CacheBackend` | Namespaced | `cb` | Yes | Bind engine Pods to typed MP; optionally provision a remote Redis provider. | | `CachePolicy` | Namespaced | `cpol` | Declarative (pushed to server) | Per-namespace lookup and eviction tuning. | | `CacheTenant` | Namespaced | `ct` | Declarative (pushed to server) | Tenant identity + entry-count quota. | | `CacheIndex` | **Cluster** | `ci` | Yes (status-only) | Cluster-wide mirror of the server aggregate. | @@ -31,17 +31,18 @@ contract follows its own compatibility policy for external consumers. |---|---|---|---| | `runtime` | `VLLM`, `SGLang` | — | Inference runtime identity. | | `type` | `LMCache`, `SGLangHiCache` | `LMCache` | Engine-side cache implementation. | -| `lmCache` | object | — | Engine-side LMCache configuration. | -| `remoteStorage` | object | — | Optional provider (`Redis`, `LMCacheServer`, `Mooncake`), ownership (`Managed`, `External`), and external endpoint. | +| `lmCache` | object | — | Typed LMCache MP topology and server configuration. Current offload uses `topology: PodLocal`. | +| `lmCache.podLocal.server.resources` | ResourceRequirements | required | Resources for the injected MP server; memory covers L1 plus 1Gi and CPU request is positive. | +| `remoteStorage` | object | — | Optional Redis L3 with `Managed` or `External` ownership. Legacy providers remain in the alpha schema only until Phase 7. | | `observation` | object | — | Model identity and first-event timeout. | | `deploymentKind` | `Deployment`, `StatefulSet` | `Deployment` | `StatefulSet` reserved/no-op. | | `replicas` | int32 | `1` | Min 0. | | `autoscaling` | object | — | `minReplicas`, `maxReplicas` (required), `targetCPUUtilizationPercent` (default 80). | | `integration.mode` | `Offload`, `EventsOnly` | `Offload` | Events-only = routing only. | -| `integration.role` | `ReadOnly`, `WriteOnly`, `ReadWrite` | `ReadWrite` | Maps to LMCache `kv_role`. | +| `integration.role` | `ReadOnly`, `WriteOnly`, `ReadWrite` | `ReadWrite` | LMCache currently admits only `ReadWrite`; directional semantics are future work. | | `integration.failOpen` | bool | `true` | `false` fails closed. | | `integration.engineOverrides` | object | — | `args` / `suppressArgs` / `env` / `suppressEnv`. | -| `integration.engineHostNetwork` | bool | `false` | Opt-in for Mooncake engine pods. | +| `integration.engineHostNetwork` | bool | `false` | Legacy engine-side Mooncake compatibility field; not used by typed MP. | | `engineSelector.matchLabels` | map | — | Equality selector over engine pod labels. | | `template` | object | — | Narrow pod-level overrides (no containers). | | `remoteStorage..resources` | ResourceRequirements | renderer default: `requests.memory 4Gi` / `limits.memory 8Gi` | Resources for the selected managed provider container. | diff --git a/site/content/en/docs/tasks/bind-an-engine.md b/site/content/en/docs/tasks/bind-an-engine.md index 448f26cf..62ed994e 100644 --- a/site/content/en/docs/tasks/bind-an-engine.md +++ b/site/content/en/docs/tasks/bind-an-engine.md @@ -3,158 +3,64 @@ title: "Bind an engine" linkTitle: "Bind an engine" weight: 2 description: > - The selector → webhook → injection lifecycle, the reserved args/env you cannot override, - and the ways binding goes wrong. + The selector → webhook → typed-MP injection lifecycle and its failure modes. --- -Binding is how a `CacheBackend` claims inference-engine pods and injects the KV-connector -wiring into them. Three actors participate: +Binding is how a `CacheBackend` claims inference-engine Pods in its namespace. +The engine Deployment and image remain inference-system owned. -- **`CacheBackend`** — its `spec.engineSelector.matchLabels` is a label selector over pods - in the same namespace, with the same semantics as `Service.spec.selector`. -- **Engine pod** — a vLLM or SGLang pod, typically owned by a user-managed Deployment. Its - `template.metadata.labels` are what the selector matches. -- **The mutating Pod webhook** — intercepts pod CREATE, matches the selector, and stamps the - engine container with the KV-connector env and CLI args (and, when enabled, the - `kvevent-subscriber` sidecar). +## Lifecycle -## The lifecycle +1. Apply a typed `CacheBackend` with `spec.lmCache.topology: PodLocal`. +2. Create engine Pods whose labels include every + `spec.engineSelector.matchLabels` entry. +3. At Pod CREATE, the webhook atomically injects the LMCache MP native sidecar, + shared memory, and the runtime-specific connector wire. +4. When configured, the `kvevent-subscriber` sidecar reports KV events to the + routing index. -1. **Apply the CacheBackend.** The reconciler provisions the managed cache-server Deployment - + Service and publishes the address in `status.endpoint`. -2. **Deploy the engine** with pod-template labels that include every key/value in - `spec.engineSelector.matchLabels`. -3. **The webhook claims matching pods** — but only once `status.endpoint` is populated. It - injects the LMCache env, the `--kv-transfer-config` arg, and stamps - `inferencecache.io/injected-by: /`. When the controller runs with - `--kvevent-subscriber-image` set **and** the backend has `spec.observation.modelID`, it also - appends the subscriber sidecar. -4. **KV events flow** (when the sidecar is present) into the server's index and surface in - `CacheBackend.status`. - -{{% alert title="The match is evaluated once, at pod CREATE" color="warning" %}} -Relabeling an existing pod does **not** re-evaluate it, and the wiring is sticky to the -pod's lifetime. To rewire after changing labels, delete the pod (the Deployment recreates -it) or `kubectl rollout restart` the Deployment. -{{% /alert %}} - -## The one-label rule (annotated) - -The selector value must appear in two places — that is the whole binding: - -```yaml -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: qwen-demo-cache # CR name — must differ from the engine Deployment name -spec: - runtime: VLLM - type: LMCache - integration: - role: ReadWrite - engineSelector: - matchLabels: - app: qwen-demo # selector key/value (1 of 2) - observation: - modelID: Qwen/Qwen2.5-0.5B-Instruct ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: qwen-engine # must differ from the CR name (the CR reconciles into a - # cache-server Deployment named after the CR — sharing names collides) -spec: - selector: - matchLabels: - app: qwen-demo - template: - metadata: - labels: - app: qwen-demo # selector key/value (2 of 2) — this is what the webhook sees - spec: - containers: - - name: vllm - image: vllm/vllm-openai-cpu:latest-x86_64 - args: ["--model", "Qwen/Qwen2.5-0.5B-Instruct"] -``` - -{{% alert title="Matched > 0 is not proof of injection" color="warning" %}} -`status.matchedEnginePods` counts pods whose *labels* match. If `status.endpoint` was empty -when a pod was admitted (engine applied before the cache server was ready), the webhook -fail-opens and the pod is admitted **unwired** — yet still counted. The authoritative wiring -signals are the per-pod `inferencecache.io/injected-by` annotation and the -`InjectedByCacheBackend` Event on the pod. Recovery is `kubectl rollout restart`. +{{% alert title="The match is evaluated once, at Pod CREATE" color="warning" %}} +Relabeling an existing Pod does not re-run admission. Recreate the Pod after +changing labels or CacheBackend configuration. {{% /alert %}} -## What gets injected, and what you can override - -The webhook injects a canonical set of args and env. Some entries are **reserved** — they -carry correctness guarantees, and `spec.integration.engineOverrides` is **hard-rejected at -admission** if it touches them. +## Typed vLLM MP wire -### vLLM + LMCache - -Always injected (reserved — not overridable): - -- `--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":""}'` - (`role` maps from `integration.role`: ReadOnly→`kv_consumer`, WriteOnly→`kv_producer`, - ReadWrite→`kv_both`) -- `LMCACHE_REMOTE_URL=lm://` -- `VLLM_USE_V1=1` -- `INFERENCECACHE_FAIL_OPEN=` -- `PYTHONHASHSEED=0` — a correctness invariant (pins the engine's hash seed so LMCache - reloads match under tensor parallelism) - -Typed LMCache tunables live under `spec.lmCache`: `chunkSizeTokens`, -`remoteSerde`, and `hostMemory.capacity`. - -### SGLang + LMCache - -Reserved args: `--enable-lmcache`. Reserved env: `LMCACHE_REMOTE_URL`, -`LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. Only `role: ReadWrite` is supported. - -{{% alert title="Known limitation — SGLang tier-2 data plane" color="warning" %}} -The `hash_scheme: sglang` routing path and the `--enable-lmcache` flag are correct, but the -current LMCache *data plane* wiring for SGLang is not — SGLang uses a node-local MP-mode -worker that reads a config file rather than the injected `LMCACHE_*` env, so a `(sglang, -LMCache)` backend can reconcile `Ready` while offloading nothing (admission emits a warning). -The MP-mode fix is designed but not yet shipped. Use vLLM for tier-2 offload today. -{{% /alert %}} +The vLLM adapter injects `LMCacheMPConnector` with the module path +`lmcache.integration.vllm.lmcache_mp_connector`, loopback host/port, +`--disable-hybrid-kv-cache-manager`, `PYTHONHASHSEED=0`, and the fail-open +value. Reserved entries are: -### Overriding safely +- `--kv-transfer-config`; +- `--disable-hybrid-kv-cache-manager`; +- `PYTHONHASHSEED`; and +- `INFERENCECACHE_FAIL_OPEN`. -```yaml -spec: - integration: - engineOverrides: - args: ["--max-model-len", "8192"] # add - suppressArgs: ["--some-default-arg"] # remove a non-reserved canonical arg - env: - - name: MY_TUNABLE - value: "1" - suppressEnv: ["SOME_DEFAULT_ENV"] -``` +## Typed SGLang MP wire -Overriding a reserved arg/env is rejected — the rejection is the point (a silently-ignored -warning would let the engine crash later with no breadcrumb). +SGLang uses `--enable-lmcache`, `--lmcache-config-file`, and +`LMCACHE_USE_EXPERIMENTAL=True`. Its reserved set is those two arguments plus +`LMCACHE_USE_EXPERIMENTAL` and `INFERENCECACHE_FAIL_OPEN`. -## Opting a pod out +Both engines share the same typed `lmcache-mp-server` renderer, but their +launch surfaces are intentionally separate. LMCache currently supports only +`integration.role: ReadWrite`. -Set `inferencecache.io/skip-inject: "true"` on the **pod template** (not the Deployment). -The pod is admitted vanilla, stamped `inferencecache.io/inject-skipped`, and gets a -`SkippedByOperator` Event so an intentional opt-out is distinguishable from selector drift. +Use `spec.lmCache.chunkSizeTokens` and +`spec.lmCache.podLocal.server` for cache configuration. Do not use +engineOverrides to replace the connector wire. ## Common failure modes | Symptom | Cause | Fix | |---|---|---| -| Engine runs uncached, `Matched: 0` | Selector and pod labels don't overlap | Reconcile the label sets on the CR or the Deployment template. | -| Two CacheBackends match one pod | Overlapping selectors | The webhook picks the lexicographically-first CR by name; narrow the selectors so each pod matches exactly one. | -| Relabeled pod still uncached | Match is CREATE-only | Delete the pod; the Deployment recreates it. | -| Old wiring after deleting the CR | Wiring is sticky to pod lifetime | Rolling-restart the engine Deployment. | -| `Matched > 0` but no injection | `status.endpoint` empty at admission | `kubectl rollout restart`; check `injected-by` annotation. | +| `MATCHED: 0` | Selector and Pod labels differ. | Align labels and recreate the Pod. | +| Matching Pod has no injection annotation | Admission failed open on a collision, invalid Pod shape, or unavailable managed endpoint. | Inspect webhook logs and Pod Events, fix the shape, recreate the Pod. | +| Engine crash-loops after injection | The runtime-owned image lacks a compatible connector/package or has another startup failure. | Inspect engine logs and select a compatible pinned image. | +| Two CacheBackends match one Pod | Selectors overlap. | Narrow selectors; the lexicographically first name otherwise wins. | -## Related pages +Set `inferencecache.io/skip-inject: "true"` on the Pod template for an +intentional opt-out. -- [CacheBackend]({{< relref "/docs/concepts/cachebackend/" >}}) — the resource in full. -- [Troubleshooting]({{< relref "/docs/administration/troubleshooting/" >}}) — the readiness runbook. +Legacy topology-less vLLM/IP injection is retained only for compatibility tests +until Phase 7 and is not a current binding recipe. diff --git a/site/content/en/docs/tasks/deploy-a-cache-backend.md b/site/content/en/docs/tasks/deploy-a-cache-backend.md index b4f3d815..2164c6b1 100644 --- a/site/content/en/docs/tasks/deploy-a-cache-backend.md +++ b/site/content/en/docs/tasks/deploy-a-cache-backend.md @@ -27,12 +27,28 @@ spec: engineSelector: matchLabels: app: my-engine # must match your engine pods' labels + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 4Gi + maxWorkers: 4 + resources: + requests: + cpu: "1" + memory: 5Gi + limits: + cpu: "2" + memory: 6Gi observation: modelID: Qwen/Qwen2.5-0.5B-Instruct ``` -Everything else is defaulted: `spec.replicas` becomes `1`, the readiness gate's -`observation.firstEventTimeout` becomes `5m`, and `integration.failOpen` is treated as `true`. +Omitting `remoteStorage` intentionally selects host-only PodLocal MP. The +readiness gate's `observation.firstEventTimeout` defaults to `5m`, and +`integration.failOpen` is treated as `true`. {{% alert title="One label does the binding" color="warning" %}} The value under `engineSelector.matchLabels` must also appear on your engine pods' template @@ -43,19 +59,18 @@ shows `MATCHED: 0`. ## 2. Add engine pods (copy a recipe) -The `CacheBackend` on its own provisions the managed cache server. For a working end-to-end -setup you also need engine pods carrying that label and publishing KV events. The fastest -path is the CPU dev recipe, which needs no GPU: +The CacheBackend injects an MP server into matching Pods but does not replace +the engine image. For a working end-to-end setup, supply a connector-compatible +image and publish KV events. A paired repository shape is: ```bash kubectl apply -f \ - https://raw.githubusercontent.com/cachebox-project/inference-cache/main/config/samples/recipe-cpu-dev.yaml + https://raw.githubusercontent.com/cachebox-project/inference-cache/main/config/samples/cachebackend-with-engine.yaml ``` -That single file ships the `CacheBackend` above plus a matching tiny-model vLLM engine -Deployment, with the engine wired to the cache. Acting on the resulting `LookupRoute` hints -to route requests is the gateway's job (which integrates separately) — so this recipe is the -cache half, not a full gateway round-trip. +That file ships a typed host-only CacheBackend plus a matching vLLM Deployment. +Normal engine startup is the authoritative connector/package compatibility +check. Acting on `LookupRoute` hints remains the gateway's job. The recipe catalog under `config/samples/` includes CPU dev, GPU production, external cache, multi-tenant, and engine-tuning scenarios. @@ -79,15 +94,15 @@ The Pod webhook runs only on CREATE. If the engine Deployment already exists, re pods after the controller rollout: ```bash -kubectl rollout restart deployment/cpu-dev-engine -kubectl rollout status deployment/cpu-dev-engine +kubectl rollout restart deployment/qwen-engine +kubectl rollout status deployment/qwen-engine ``` Finally, send one request so vLLM publishes the first KV event. Keep the port-forward running in one terminal: ```bash -kubectl port-forward deployment/cpu-dev-engine 8000:8000 +kubectl port-forward deployment/qwen-engine 8000:8000 ``` Then call the engine from another: @@ -107,8 +122,8 @@ External backends are exempt from this gate. ``` $ kubectl get cachebackend -NAME TYPE READY MATCHED ENDPOINT PREFIXES LASTEVENT AGE -my-cache LMCache True 1 my-cache.default... 128 12s 3m +NAME TYPE READY MATCHED ENDPOINT PREFIXES LASTEVENT AGE +my-cache LMCache True 1 128 12s 3m ``` - `MATCHED` — the number of engine pods the selector binds. @@ -138,8 +153,8 @@ Once the backend is Ready and engine pods are bound, three things are live: - **Cache-aware routing** — the server answers `LookupRoute` with which replicas hold which prefixes warm, so a gateway can route for a prefix cache hit. -- **KV reuse** — matched engine pods get the LMCache wiring injected automatically, so their - KV cache is offloaded to and reused from the managed backend. +- **KV reuse** — matched engine Pods get the typed MP wiring injected + automatically; host-only L1 is per Pod, and Redis L3 is optional. - **Observability** — `kubectl get cachebackend` and the cluster-wide `CacheIndex` surface live state. From 4c09a87a5bdf44a9a8850259ccc233820fb37dde Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Tue, 11 Aug 2026 17:49:39 -0700 Subject: [PATCH 07/13] Remove legacy LMCache IP data plane Make typed multiprocess the only LMCache production path, trim legacy API and lifecycle surfaces, preserve managed-provider workload controls, and add upgrade/GPU regression evidence. Signed-off-by: Yue Sun --- .github/workflows/c2-reconciler-canary.yml | 65 - .github/workflows/c6-engine-wiring-canary.yml | 70 - .github/workflows/default-install-smoke.yml | 26 +- .github/workflows/phase5-upgrade-smoke.yml | 50 + README.md | 6 +- api/v1alpha1/cachebackend_effective_test.go | 2 +- api/v1alpha1/cachebackend_types.go | 335 +- api/v1alpha1/cachebackend_types_test.go | 352 +- api/v1alpha1/zz_generated.deepcopy.go | 183 +- cmd/controller/main.go | 12 +- cmd/inferencecache/doctor.go | 6 +- cmd/inferencecache/doctor_unit_test.go | 1 - .../inferencecache.io_cachebackends.yaml | 2668 +++++----- config/manager/manager.yaml | 1 - config/observability/kustomization.yaml | 4 +- config/observability/podmonitor.yaml | 6 +- .../gpu-validation/kustomization.yaml | 3 - config/rbac/role.yaml | 12 - ...ebackend-invalid-scale-to-zero-no-min.yaml | 31 - config/samples/cachebackend-cpu-override.yaml | 2 - config/samples/cachebackend-events-only.yaml | 13 +- config/samples/cachebackend-external.yaml | 4 +- config/samples/cachebackend-lmcache.yaml | 7 +- .../samples/cachebackend-with-override.yaml | 2 - config/samples/recipe-cpu-dev.yaml | 1 - config/samples/recipe-external-cache.yaml | 2 +- config/samples/recipe-gpu-production.yaml | 2 +- config/samples/recipe-multi-tenant.yaml | 2 - config/samples/recipe-tuning.yaml | 1 - docs/cli/doctor.md | 8 +- docs/design/cachebackend-api.md | 404 +- docs/design/crd-contract.md | 2 +- docs/design/kvevent-subscriber-wiring.md | 11 +- .../lmcache-multiprocess-migration-roadmap.md | 270 +- docs/design/sglang-lmcache-mp-mode.md | 108 +- docs/observability/alerts.md | 7 +- docs/quickstart.md | 5 +- docs/reference-stack/README.md | 19 +- docs/reference-stack/VERSIONS.md | 11 +- .../manifests/sglang-lmcache/README.md | 7 +- .../manifests/sglang-lmcache/deployment.yaml | 5 +- .../scripts/canary_c2_reconcile.sh | 193 - .../scripts/canary_c6_engine_wiring.sh | 308 -- .../scripts/default_install_smoke.sh | 4482 +---------------- .../scripts/phase5_upgrade_smoke.sh | 206 + docs/reference/metrics.md | 17 +- internal/adapters/builtin/registry.go | 6 +- internal/adapters/builtin/registry_test.go | 112 +- .../builtin/runtime/lmcache_mp_renderer.go | 4 - .../runtime/lmcache_mp_renderer_test.go | 6 +- .../adapters/builtin/runtime/lmcachecheck.go | 2 +- .../builtin/runtime/lmcachecheck_test.go | 18 +- .../builtin/runtime/runtime_helpers.go | 232 + .../runtime/runtime_helpers_unit_test.go | 156 + .../builtin/runtime/sglang_hicache.go | 3 - .../builtin/runtime/sglang_hicache_test.go | 3 - .../builtin/runtime/sglang_lmcache.go | 48 +- .../builtin/runtime/sglang_lmcache_test.go | 1178 +---- .../builtin/runtime/sglang_lmcache_wire.go | 680 --- .../builtin/runtime/test_helpers_test.go | 70 + .../adapters/builtin/runtime/vllm_lmcache.go | 213 - .../builtin/runtime/vllm_lmcache_mp.go | 60 +- .../builtin/runtime/vllm_lmcache_mp_test.go | 81 +- .../builtin/runtime/vllm_lmcache_test.go | 1398 ----- .../builtin/runtime/vllm_lmcache_wire.go | 503 -- .../builtin/runtime/vllm_lmcache_wire_test.go | 363 -- .../builtin/storage/effective_config.go | 63 +- .../builtin/storage/lmcache_server.go | 109 - internal/adapters/builtin/storage/mooncake.go | 114 - internal/adapters/builtin/storage/redis.go | 17 +- .../adapters/builtin/storage/redis_test.go | 19 +- internal/adapters/builtin/storage/registry.go | 42 +- .../adapters/builtin/storage/registry_test.go | 177 +- internal/cli/doctor/checks/cachebackend.go | 12 +- internal/cli/doctor/checks/checks.go | 2 +- internal/cli/doctor/checks/checks_test.go | 34 +- internal/cli/doctor/finding.go | 2 +- .../cachebackend_autoscaling_test.go | 516 -- internal/controller/cachebackend_dispatch.go | 16 +- ...chebackend_events_only_integration_test.go | 45 +- .../controller/cachebackend_events_test.go | 137 +- .../cachebackend_hostnetwork_test.go | 152 - .../cachebackend_kvevent_gate_test.go | 43 +- .../cachebackend_lmcache_mp_status.go | 4 + .../cachebackend_lmcache_mp_status_test.go | 44 - internal/controller/cachebackend_managed.go | 98 +- .../controller/cachebackend_managed_test.go | 180 +- .../cachebackend_matched_pods_test.go | 18 +- ...d_mooncake_hostnetwork_integration_test.go | 293 -- .../cachebackend_mp_lifecycle_test.go | 89 + internal/controller/cachebackend_probe.go | 4 +- .../cachebackend_probe_integration_test.go | 5 +- .../controller/cachebackend_reconciler.go | 35 +- .../cachebackend_reconciler_test.go | 68 +- ...cachebackend_resources_integration_test.go | 36 +- ...chebackend_schema_trim_integration_test.go | 79 +- .../controller/cachebackend_server_restart.go | 1290 ----- ...backend_server_restart_integration_test.go | 432 -- .../cachebackend_server_restart_test.go | 1873 ------- .../controller/cachebackend_serverless.go | 62 +- .../cachebackend_serverless_test.go | 804 +-- internal/controller/cachebackend_status.go | 50 +- .../controller/cachebackend_status_test.go | 48 +- .../cachebackend_t2degraded_test.go | 2 +- internal/controller/cachebackend_workload.go | 367 +- .../controller/cachebackend_workload_test.go | 125 +- .../contract_coverage_sweep_test.go | 36 +- internal/controller/envtest_helpers_test.go | 93 + internal/controller/integration_test.go | 1830 ------- internal/enginebinding/runtime.go | 6 - internal/webhook/pod/doc.go | 4 +- .../webhook/pod/envtest_integration_test.go | 90 +- internal/webhook/pod/podinjector.go | 43 +- internal/webhook/pod/podinjector_test.go | 815 +-- .../v1alpha1/cachebackend_defaulter.go | 44 +- .../cachebackend_defaulter_envtest_test.go | 268 +- .../v1alpha1/cachebackend_defaulter_test.go | 167 +- .../cachebackend_integration_validation.go | 210 - ...achebackend_integration_validation_test.go | 959 ---- .../cachebackend_lmcache_mp_validation.go | 63 +- ...cachebackend_lmcache_mp_validation_test.go | 379 +- .../cachebackend_override_validation_test.go | 191 +- .../cachebackend_storage_validation.go | 97 +- .../cachebackend_storage_validation_test.go | 710 +-- .../v1alpha1/cachebackend_validator.go | 9 +- .../v1alpha1/cachebackend_validator_test.go | 470 +- pkg/adapters/backend/backend.go | 8 +- pkg/adapters/backend/backend_test.go | 4 +- pkg/adapters/backend/endpoint.go | 95 +- pkg/adapters/runtime/adapter.go | 8 +- pkg/adapters/runtime/adapter_test.go | 2 +- site/content/en/docs/concepts/_index.md | 6 +- site/content/en/docs/concepts/architecture.md | 5 +- site/content/en/docs/concepts/cachebackend.md | 17 +- site/content/en/docs/reference/crd-api.md | 10 +- site/content/en/docs/reference/metrics.md | 1 - 136 files changed, 4457 insertions(+), 24757 deletions(-) delete mode 100644 .github/workflows/c2-reconciler-canary.yml delete mode 100644 .github/workflows/c6-engine-wiring-canary.yml create mode 100644 .github/workflows/phase5-upgrade-smoke.yml delete mode 100644 config/samples/_test/cachebackend-invalid-scale-to-zero-no-min.yaml delete mode 100755 docs/reference-stack/scripts/canary_c2_reconcile.sh delete mode 100755 docs/reference-stack/scripts/canary_c6_engine_wiring.sh create mode 100755 docs/reference-stack/scripts/phase5_upgrade_smoke.sh create mode 100644 internal/adapters/builtin/runtime/runtime_helpers.go create mode 100644 internal/adapters/builtin/runtime/runtime_helpers_unit_test.go delete mode 100644 internal/adapters/builtin/runtime/sglang_lmcache_wire.go create mode 100644 internal/adapters/builtin/runtime/test_helpers_test.go delete mode 100644 internal/adapters/builtin/runtime/vllm_lmcache.go delete mode 100644 internal/adapters/builtin/runtime/vllm_lmcache_test.go delete mode 100644 internal/adapters/builtin/runtime/vllm_lmcache_wire.go delete mode 100644 internal/adapters/builtin/runtime/vllm_lmcache_wire_test.go delete mode 100644 internal/adapters/builtin/storage/lmcache_server.go delete mode 100644 internal/adapters/builtin/storage/mooncake.go delete mode 100644 internal/controller/cachebackend_autoscaling_test.go delete mode 100644 internal/controller/cachebackend_hostnetwork_test.go delete mode 100644 internal/controller/cachebackend_mooncake_hostnetwork_integration_test.go create mode 100644 internal/controller/cachebackend_mp_lifecycle_test.go delete mode 100644 internal/controller/cachebackend_server_restart.go delete mode 100644 internal/controller/cachebackend_server_restart_integration_test.go delete mode 100644 internal/controller/cachebackend_server_restart_test.go create mode 100644 internal/controller/envtest_helpers_test.go delete mode 100644 internal/controller/integration_test.go delete mode 100644 internal/webhook/v1alpha1/cachebackend_integration_validation_test.go diff --git a/.github/workflows/c2-reconciler-canary.yml b/.github/workflows/c2-reconciler-canary.yml deleted file mode 100644 index 83360b22..00000000 --- a/.github/workflows/c2-reconciler-canary.yml +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# Manual legacy-IP compatibility canary for the C2 CacheBackend reconciler. -# It intentionally exercises remoteStorage.provider=LMCacheServer until Phase 7; -# it is not a current deployment reference and no longer runs on a schedule. -# -# Runs docs/reference-stack/scripts/canary_c2_reconcile.sh: brings up a kind -# cluster, runs the controller, applies a CPU-profile CacheBackend, and asserts -# the reconciler stands up a healthy serving backend (Ready condition True, -# endpoint published) and owner-ref GC on delete. The optional traffic block -# (engine prefix-cache hit through a port-forwarded Service) is opt-in via -# SKIP_TRAFFIC=0 — see the script header for what an engine-paired run needs. -# GPU-free. -# -# This is NOT a per-PR gate (it pulls a multi-GB image, needs Docker + kind, and -# ~10 GiB RAM). -name: legacy-ip-c2-reconciler-canary - -on: - workflow_dispatch: - inputs: - runner: - description: "Runner label (override to target a self-hosted Docker host)" - default: ubuntu-latest - required: false - -permissions: - contents: read - -concurrency: - group: legacy-ip-c2-reconciler-canary - cancel-in-progress: false - -jobs: - canary: - runs-on: ${{ github.event.inputs.runner || 'ubuntu-latest' }} - timeout-minutes: 40 - permissions: - contents: read - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 - with: - go-version-file: go.mod - - - name: Install kind - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 - with: - install_only: true - - - name: Run C2 reconciler CPU canary - run: docs/reference-stack/scripts/canary_c2_reconcile.sh - - - name: Upload canary logs on failure - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: c2-canary-logs - path: | - /tmp/c2-canary-controller.log - /tmp/c2-canary-pf.log - if-no-files-found: ignore diff --git a/.github/workflows/c6-engine-wiring-canary.yml b/.github/workflows/c6-engine-wiring-canary.yml deleted file mode 100644 index 836f648b..00000000 --- a/.github/workflows/c6-engine-wiring-canary.yml +++ /dev/null @@ -1,70 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# Manual legacy-IP compatibility canary for the C6 vLLM+LMCache engine-pod -# wiring webhook + cross-pod cache reuse. It intentionally exercises -# LMCacheConnectorV1 and LMCacheServer until Phase 7; it is not a current -# deployment reference and no longer runs on a schedule. -# -# Runs docs/reference-stack/scripts/canary_c6_engine_wiring.sh: brings up -# a kind cluster, installs cert-manager + this repo's config/default -# (controller + mutating Pod webhook + lmcache wiring), applies a -# CacheBackend, then creates two vLLM CPU engine pods labeled to match the -# CacheBackend.Spec.EngineSelector. Asserts the webhook injects the -# LMCache wiring on both pods at admission and (when traffic is enabled) -# that engine-b reports a vllm:prefix_cache_hits increment populated by a -# prompt prefix engine-a previously serviced via the shared lmcache-server. -# -# This is NOT a per-PR gate (multi-GB image pull, ~12 GiB RAM for two CPU -# engines + lmcache-server + cert-manager). -name: legacy-ip-c6-engine-wiring-canary - -on: - workflow_dispatch: - inputs: - runner: - description: "Runner label (override to target a self-hosted Docker host)" - default: ubuntu-latest - required: false - skip_traffic: - description: "Skip the traffic-driving step (still asserts webhook wiring)" - default: "0" - required: false - -permissions: - contents: read - -concurrency: - group: legacy-ip-c6-engine-wiring-canary - cancel-in-progress: false - -jobs: - canary: - runs-on: ${{ github.event.inputs.runner || 'ubuntu-latest' }} - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Install kind - uses: helm/kind-action@v1 - with: - install_only: true - - - name: Run C6 engine-wiring CPU canary - env: - SKIP_TRAFFIC: ${{ github.event.inputs.skip_traffic || '0' }} - run: docs/reference-stack/scripts/canary_c6_engine_wiring.sh - - - name: Upload canary logs on failure - if: failure() - uses: actions/upload-artifact@v4 - with: - name: c6-canary-logs - path: | - /tmp/c6-canary-*.log - if-no-files-found: ignore diff --git a/.github/workflows/default-install-smoke.yml b/.github/workflows/default-install-smoke.yml index d67b9c06..e677f76e 100644 --- a/.github/workflows/default-install-smoke.yml +++ b/.github/workflows/default-install-smoke.yml @@ -12,16 +12,16 @@ # - cacheindex/cluster-default.status.observedServer is populated (proves the # controller's CacheIndex poller is talking to the server's /snapshot) # - gRPC LookupRoute returns reason_code=NO_HINT (fail-open default) -# - typed PodLocal MP injection for vLLM and SGLang, including native sidecar, -# connector arguments, shared memory, and optional Redis L3 rendering -# - explicitly labelled legacy-IP compatibility checks for the implementation -# retained until Phase 7; these inline fixtures are not deployment examples +# - server /readyz and /metrics are reachable through the installed Service +# - current samples pass live server-side admission, and the generic +# CachePolicy/CacheTenant/PromptTemplate/PDTopology APIs remain usable +# - the served CacheBackend CRD contains only the MP schema +# - typed PodLocal MP admission injects the vLLM connector and native sidecar +# - managed Redis renders independently and publishes remote-storage status +# - an idempotent bundle re-apply preserves typed CacheBackend objects # -# Lightweight (two distroless ~30 MB images + cert-manager + a busybox -# stand-in for the engine container + a pause-image pod, no real engine -# pull), so it runs on every PR. Sister-canaries -# (legacy-ip-c2-reconciler-canary, legacy-ip-c6-engine-wiring-canary) remain -# manual compatibility checks until Phase 7. +# Lightweight: no real inference engine, LMCache server, GPU, or model is +# started, so it runs on every PR. name: default-install-smoke on: @@ -41,11 +41,9 @@ concurrency: jobs: install-smoke: runs-on: ubuntu-latest - # Script targets ~6 min end-to-end (incl. the Calico CNI install the smoke - # needs to enforce NetworkPolicy). The timeout is a circuit breaker for a - # wedged kind cluster / image pull, not the expected runtime; it sits above - # the worst-case Calico readiness budget (~510s) plus the smoke so a wedged - # CNI still leaves room for the exit trap to collect diagnostics. + # The timeout is a circuit breaker for a wedged kind cluster, image build, + # cert-manager rollout, or admission check; the exit trap still has time to + # collect diagnostics. timeout-minutes: 20 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 diff --git a/.github/workflows/phase5-upgrade-smoke.yml b/.github/workflows/phase5-upgrade-smoke.yml new file mode 100644 index 00000000..aa8af5a6 --- /dev/null +++ b/.github/workflows/phase5-upgrade-smoke.yml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +name: phase5-upgrade-smoke + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + typed-object-upgrade: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + + - name: Install kind + uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 + with: + install_only: true + + - name: Upgrade Phase 5 typed objects to Phase 7 + env: + TAG: ${{ github.sha }} + run: docs/reference-stack/scripts/phase5_upgrade_smoke.sh + + - name: Upload upgrade-smoke logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: phase5-upgrade-smoke-logs + path: /tmp/phase5-upgrade-smoke-logs/ + if-no-files-found: ignore diff --git a/README.md b/README.md index 00b002f5..d5a7948f 100644 --- a/README.md +++ b/README.md @@ -206,8 +206,7 @@ kubectl get cacheindex cluster-default -o yaml Both binaries expose Prometheus metrics on their pod's `:8080/metrics` (prefixed `inferencecache_*`) — the server binary's series cover the in-memory index and gRPC handlers; the controller binary's series cover -the reconcilers (e.g. `inferencecache_backend_probe_result_total`, -`inferencecache_backend_server_restart_cascades_total`). A default +the reconcilers (for example, `inferencecache_backend_probe_result_total`). A default alert bundle for the operational silent-failure patterns this code has hit in production ships under [`config/observability/`](config/observability/) and is **not** included @@ -262,8 +261,7 @@ use the flat [`alerting-rules.yaml`](config/observability/alerting-rules.yaml). **You must also configure scraping yourself for the server, the controller pod, and every injected PodLocal LMCache sidecar.** The server's `:8080` exposes the index, lookup, and auth series; the controller pod's `:8080` exposes the per-stage -probe-result counter (`inferencecache_backend_probe_result_total`) -and the cache-server restart-cascade counter; each LMCache sidecar exposes +probe-result counter (`inferencecache_backend_probe_result_total`); each LMCache sidecar exposes its own `lmcache_mp_*` series on `:8080/metrics` — the controller-side alerts (`ServerProbeFail` today) load against the controller's series, so a server-only scrape leaves them inert. diff --git a/api/v1alpha1/cachebackend_effective_test.go b/api/v1alpha1/cachebackend_effective_test.go index 311e3cf4..9c6f2e31 100644 --- a/api/v1alpha1/cachebackend_effective_test.go +++ b/api/v1alpha1/cachebackend_effective_test.go @@ -13,7 +13,7 @@ func TestEffectiveRemoteStorageUsesOnlyExplicitDeclaration(t *testing.T) { } want := &CacheBackendRemoteStorageSpec{ - Provider: CacheBackendRemoteStorageProviderMooncake, + Provider: CacheBackendRemoteStorageProviderRedis, Ownership: CacheBackendRemoteStorageOwnershipManaged, } spec.RemoteStorage = want diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index b27681d2..ed336113 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -32,17 +32,13 @@ const ( CacheBackendTypeSGLangHiCache CacheBackendType = "SGLangHiCache" ) -// +kubebuilder:validation:Enum=Redis;LMCacheServer;Mooncake +// +kubebuilder:validation:Enum=Redis // CacheBackendRemoteStorageProvider identifies the technology used for the // optional shared/remote cache tier. type CacheBackendRemoteStorageProvider string -const ( - CacheBackendRemoteStorageProviderRedis CacheBackendRemoteStorageProvider = "Redis" - CacheBackendRemoteStorageProviderLMCacheServer CacheBackendRemoteStorageProvider = "LMCacheServer" - CacheBackendRemoteStorageProviderMooncake CacheBackendRemoteStorageProvider = "Mooncake" -) +const CacheBackendRemoteStorageProviderRedis CacheBackendRemoteStorageProvider = "Redis" // +kubebuilder:validation:Enum=Managed;External @@ -77,16 +73,6 @@ const ( LMCacheConnectorModeMultiprocess LMCacheConnectorMode = "Multiprocess" ) -// +kubebuilder:validation:Enum=Deployment;StatefulSet - -// CacheBackendDeploymentKind identifies the Kubernetes workload kind used for managed backends. -type CacheBackendDeploymentKind string - -const ( - CacheBackendDeploymentKindDeployment CacheBackendDeploymentKind = "Deployment" - CacheBackendDeploymentKindStatefulSet CacheBackendDeploymentKind = "StatefulSet" -) - // +kubebuilder:validation:Enum=ReadOnly;WriteOnly;ReadWrite // CacheBackendIntegrationRole identifies how an engine should interact with the cache backend. @@ -188,15 +174,6 @@ type SGLangHiCacheSpec struct { MemoryLayout SGLangHiCacheMemoryLayout `json:"memoryLayout,omitempty"` } -// CacheBackendHostMemorySpec configures engine-side host memory. Capacity is -// owned by the engine cache implementation and never sizes a remote provider. -type CacheBackendHostMemorySpec struct { - // Capacity is the memory budget for the engine-side host cache. - // +optional - // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="capacity must be greater than zero" - Capacity *resource.Quantity `json:"capacity,omitempty"` -} - // LMCachePodLocalServerSpec configures the CacheBackend-owned LMCache MP // server injected into each selected engine Pod. type LMCachePodLocalServerSpec struct { @@ -283,10 +260,7 @@ type LMCacheNodeLocalSpec struct { type LMCacheEngineSpec struct { // Topology selects the canonical LMCache MP server placement. PodLocal is // implemented first; NodeLocal is reserved and rejected until Phase 8. - // Omit this field only for a legacy in-process/flat-field object during the - // repository migration window. - // +optional - Topology LMCacheTopology `json:"topology,omitempty"` + Topology LMCacheTopology `json:"topology"` // PodLocal configures one MP server in each selected engine Pod. // +optional @@ -301,27 +275,6 @@ type LMCacheEngineSpec struct { // +optional // +kubebuilder:validation:Minimum=1 ChunkSizeTokens *int32 `json:"chunkSizeTokens,omitempty"` - - // HostMemory is a legacy flat-field input retained only while repository - // consumers migrate to podLocal.server.l1Capacity. - // +optional - HostMemory *CacheBackendHostMemorySpec `json:"hostMemory,omitempty"` - - // WorkerImage is a legacy flat-field input retained only while repository - // consumers migrate to podLocal.server.image. - // +optional - WorkerImage string `json:"workerImage,omitempty"` - - // WorkerPort is a legacy flat-field input retained only while repository - // consumers migrate to podLocal.server.port. - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=65535 - WorkerPort *int32 `json:"workerPort,omitempty"` - - // RemoteSerde is a legacy in-process input and is forbidden with MP. - // +optional - RemoteSerde string `json:"remoteSerde,omitempty"` } // RemoteStorageTLSSpec configures server-authenticated TLS for a remote L3. @@ -373,39 +326,56 @@ type RedisRemoteStorageSpec struct { Database *int32 `json:"database,omitempty"` } -// LMCacheServerRemoteStorageSpec configures a standalone lmcache-server -// remote-storage provider. -type LMCacheServerRemoteStorageSpec struct { - // Image is used only when ownership is Managed. +// CacheBackendManagedWorkloadSpec configures Pod scheduling and security for a +// controller-managed remote-storage workload. Provider topology and scaling do +// not belong here: each provider must expose those semantics through its own +// typed configuration rather than treating replicas as interchangeable. +type CacheBackendManagedWorkloadSpec struct { + // NodeSelector constrains provider Pods to nodes with matching labels. // +optional - Image string `json:"image,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` - // Command overrides the managed server command and arguments. + // Affinity configures provider Pod scheduling affinity. // +optional - // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:items:MinLength=1 - Command []string `json:"command,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` - // Resources are applied to the managed lmcache-server container. + // Tolerations allow provider Pods to schedule onto tainted nodes. // +optional - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` -} + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` -// MooncakeRemoteStorageSpec configures a Mooncake remote-storage provider. -type MooncakeRemoteStorageSpec struct { - // Image is used only when ownership is Managed. + // TopologySpreadConstraints configures provider Pod spreading across + // topology domains. // +optional - Image string `json:"image,omitempty"` + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` - // Command overrides the managed Mooncake master command and arguments. + // ImagePullSecrets references Secrets used to pull provider images. // +optional - // +kubebuilder:validation:MinItems=1 - // +kubebuilder:validation:items:MinLength=1 - Command []string `json:"command,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - // Resources are applied to the managed Mooncake master container. + // ServiceAccountName is the ServiceAccount used by provider Pods. // +optional - Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // SecurityContext configures Pod-level security settings for provider Pods. + // +optional + SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` + + // PriorityClassName is the priority class assigned to provider Pods. + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // SchedulerName selects the scheduler used for provider Pods. + // +optional + SchedulerName string `json:"schedulerName,omitempty"` + + // RuntimeClassName selects the runtime class used for provider Pods. + // +optional + RuntimeClassName *string `json:"runtimeClassName,omitempty"` + + // TerminationGracePeriodSeconds configures graceful provider shutdown. + // +optional + // +kubebuilder:validation:Minimum=0 + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` } // CacheBackendRemoteStorageSpec configures the optional shared/remote tier. @@ -424,17 +394,14 @@ type CacheBackendRemoteStorageSpec struct { // +optional Endpoint string `json:"endpoint,omitempty"` - // Redis contains Redis-owned configuration. - // +optional - Redis *RedisRemoteStorageSpec `json:"redis,omitempty"` - - // LMCacheServer contains standalone lmcache-server-owned configuration. + // Workload configures Pod scheduling and security for controller-managed + // provider workloads. It is rejected when ownership is External. // +optional - LMCacheServer *LMCacheServerRemoteStorageSpec `json:"lmCacheServer,omitempty"` + Workload *CacheBackendManagedWorkloadSpec `json:"workload,omitempty"` - // Mooncake contains Mooncake-owned configuration. + // Redis contains Redis-owned configuration. // +optional - Mooncake *MooncakeRemoteStorageSpec `json:"mooncake,omitempty"` + Redis *RedisRemoteStorageSpec `json:"redis,omitempty"` } // CacheBackendObservationSpec configures KV-event observation independently @@ -451,9 +418,6 @@ type CacheBackendObservationSpec struct { } // CacheBackendSpec defines the desired state of a cache backend. -// -// The autoscaling spec (spec.autoscaling) is reconciled into a -// HorizontalPodAutoscaler for managed backends. type CacheBackendSpec struct { // Runtime identifies the inference runtime. Values are case-sensitive: use // VLLM or SGLang. @@ -483,39 +447,6 @@ type CacheBackendSpec struct { // +optional Observation *CacheBackendObservationSpec `json:"observation,omitempty"` - // DeploymentKind identifies whether a managed backend is reconciled as a - // Deployment or StatefulSet. Defaults to Deployment — the only kind the - // Phase-1 reconciler templates; StatefulSet is reserved for future - // per-replica-PVC topologies and is a no-op today. - // +optional - // +kubebuilder:default=Deployment - DeploymentKind CacheBackendDeploymentKind `json:"deploymentKind,omitempty"` - - // Replicas is the desired number of backend workload replicas. Defaults - // to 1 — a conservative single-replica deployment; operators opt into - // horizontal scale via spec.autoscaling. - // - // When spec.autoscaling is set, the HPA owns the live replica count and - // the autoscaling floor (spec.autoscaling.minReplicas) is auto-defaulted - // to spec.replicas on FIRST APPLY ONLY by the admission defaulter. - // Subsequent edits to spec.replicas do NOT move the autoscaling floor — - // minReplicas is operator-owned (and operator-pinned via the apiserver - // field manager) after first apply, matching the standard Kubernetes HPA - // convention that scaling intent flows through HPA fields once an HPA - // owns the workload. To widen or narrow the autoscaling band post-apply, - // edit spec.autoscaling.minReplicas directly. - // +optional - // +kubebuilder:default=1 - // +kubebuilder:validation:Minimum=0 - Replicas *int32 `json:"replicas,omitempty"` - - // Autoscaling configures horizontal autoscaling for the managed backend - // workload. When set, the controller reconciles a HorizontalPodAutoscaler - // owned by this CacheBackend; the HPA then drives the underlying workload's - // replica count, overriding spec.replicas. - // +optional - Autoscaling *CacheBackendAutoscalingSpec `json:"autoscaling,omitempty"` - // Integration describes how inference engines should use the cache backend. // +optional Integration *CacheBackendIntegrationSpec `json:"integration,omitempty"` @@ -526,18 +457,17 @@ type CacheBackendSpec struct { // metav1.LabelSelector surface (matchExpressions, operator-based // selection) is NOT exposed today — only MatchLabels. // Pods that match get runtime-adapter engine wiring injected by the - // mutating Pod admission webhook at pod CREATE time. Server-backed - // adapters require status.endpoint to be published first; the webhook - // fail-opens when it is empty. Engine-local adapters such as native - // SGLang HiCache, and the LMCache host-only path, explicitly require no - // endpoint and inject immediately. + // mutating Pod admission webhook at pod CREATE time. LMCache MP adapters + // inject the local connector immediately; when an optional managed Redis + // tier is configured, its address is read from status.remoteStorage.endpoint. + // Engine-local adapters such as native SGLang HiCache require no endpoint. // Admission is CREATE-only; recovery or a configuration update requires // recreating the pod (e.g. `kubectl rollout restart`), not editing its // live labels. // // This describes the default spec.integration.mode=Offload path. For // spec.integration.mode=EventsOnly no KV connector wiring (env vars + - // CLI args) is injected and status.endpoint is neither required nor + // CLI args) is injected and no connector or remote-storage endpoint is // published — the kvevent-subscriber observation sidecar alone is // injected (see below), so matched pods report cache state for routing // without offloading KV to a backend server. @@ -572,10 +502,6 @@ type CacheBackendSpec struct { // +optional HiCache *SGLangHiCacheSpec `json:"hiCache,omitempty"` - // Template provides pod-level overrides for managed backend workloads. - // +optional - Template *CacheBackendPodSpecOverride `json:"template,omitempty"` - // AllowCrossNamespace opts the CacheBackend into referencing an Endpoint // that resolves into a Kubernetes Service in a different namespace from // this object. Without this opt-in admission rejects such Endpoints, @@ -586,42 +512,6 @@ type CacheBackendSpec struct { AllowCrossNamespace bool `json:"allowCrossNamespace,omitempty"` } -// CacheBackendAutoscalingSpec configures horizontal autoscaling of the managed -// backend workload via a HorizontalPodAutoscaler. Cache-aware (custom-metric) -// autoscaling is deferred to a later module; Phase 1 supports a CPU-utilization -// target, which is sufficient to demonstrate scale-up under load. -// -// +kubebuilder:validation:XValidation:rule="!has(self.minReplicas) || self.minReplicas <= self.maxReplicas",message="minReplicas must not exceed maxReplicas" -type CacheBackendAutoscalingSpec struct { - // MinReplicas is the lower bound for the HPA replica count. The - // admission defaulter computes the default at write time from - // spec.replicas (which itself defaults to 1) so the HPA's floor matches - // the operator-declared baseline rather than a hard-coded constant. This - // is a FIRST-APPLY-ONLY default: the defaulter never overwrites an - // operator-set value, AND once stamped the field is owned by the - // apiserver field manager — subsequent edits to spec.replicas do NOT - // recompute or move minReplicas, matching the standard Kubernetes HPA - // convention that scaling intent flows through HPA fields once an HPA - // owns the workload. To widen or narrow the autoscaling band post-apply, - // edit spec.autoscaling.minReplicas directly. Operators who want a - // non-default floor on first apply set the field explicitly. - // +optional - // +kubebuilder:validation:Minimum=1 - MinReplicas *int32 `json:"minReplicas,omitempty"` - - // MaxReplicas is the upper bound for the HPA replica count. - // +kubebuilder:validation:Required - // +kubebuilder:validation:Minimum=1 - MaxReplicas int32 `json:"maxReplicas"` - - // TargetCPUUtilizationPercent is the average per-pod CPU utilization the - // HPA targets. Defaults to 80 when unset. - // +optional - // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:Maximum=100 - TargetCPUUtilizationPercent *int32 `json:"targetCPUUtilizationPercent,omitempty"` -} - // CacheBackendIntegrationSpec describes engine integration behavior. // // Per-namespace lookup tuning lives on CachePolicy, not here: the lookup @@ -640,8 +530,8 @@ type CacheBackendIntegrationSpec struct { // EventsOnly is the supported integration for hybrid-attention models that // cannot take a vLLM KV connector (and a lighter routing-only deployment for // anyone who does not want an offload tier). Because EventsOnly provisions - // no server, status.endpoint stays empty and the autoscaling spec is - // rejected at admission; Ready is still gated on the first observed KV event. + // no server, connector and remote-storage status stay empty; Ready is still + // gated on the first observed KV event. // SGLangHiCache supports Offload only and is rejected with EventsOnly. // See the CacheBackendIntegrationMode godoc. // +optional @@ -702,30 +592,6 @@ type CacheBackendIntegrationSpec struct { // crashed engine. See the package doc for the rationale. // +optional EngineOverrides *EngineInjectionOverrides `json:"engineOverrides,omitempty"` - - // EngineHostNetwork opts engine pods bound to this backend into host - // networking. It exists for exactly one backend today: Mooncake, whose - // transfer engine is a peer-to-peer mesh — the engine dials a real node IP - // on a dynamically negotiated port, which a CNI overlay pod IP cannot do. - // Without this the backend reconciles Ready and transfers zero KV, and - // admission warns as much on every apply. - // - // This is opt-in, and deliberately not a default, because it rewrites the - // networking of a pod the operator owns: - // - hostNetwork is a privilege. A Pod Security "restricted" namespace - // rejects such a pod, and because mutating webhooks run BEFORE Pod - // Security validation, silently injecting it would turn a working engine - // pod into a rejected one — with an error that names Pod Security, not - // this controller. - // - The pod's ports move onto the node's interfaces, outside the pod - // network. NetworkPolicy selects pods by pod IP and therefore stops - // constraining them (see docs/design/cachebackend-api.md). - // - // Admission rejects this on any backend type that does not need it, so it - // can never sit inert on a CacheBackend. - // - // +optional - EngineHostNetwork bool `json:"engineHostNetwork,omitempty"` } // EngineInjectionOverrides is the in-between knob between "take the @@ -842,54 +708,6 @@ type CacheBackendEngineSelector struct { MatchLabels map[string]string `json:"matchLabels,omitempty"` } -// CacheBackendPodSpecOverride defines optional pod-level overrides applied to managed backend pods. -type CacheBackendPodSpecOverride struct { - // NodeSelector constrains backend pods to nodes with matching labels. - // +optional - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - - // Affinity configures backend pod scheduling affinity. - // +optional - Affinity *corev1.Affinity `json:"affinity,omitempty"` - - // Tolerations allow backend pods to schedule onto tainted nodes. - // +optional - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - - // TopologySpreadConstraints configures backend pod spreading across topology domains. - // +optional - TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` - - // ImagePullSecrets references secrets used to pull backend pod images. - // +optional - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - - // ServiceAccountName is the service account used by backend pods. - // +optional - ServiceAccountName string `json:"serviceAccountName,omitempty"` - - // SecurityContext configures pod-level security settings for backend pods. - // +optional - SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` - - // PriorityClassName is the priority class assigned to backend pods. - // +optional - PriorityClassName string `json:"priorityClassName,omitempty"` - - // SchedulerName selects the scheduler used for backend pods. - // +optional - SchedulerName string `json:"schedulerName,omitempty"` - - // RuntimeClassName selects the runtime class used for backend pods. - // +optional - RuntimeClassName *string `json:"runtimeClassName,omitempty"` - - // TerminationGracePeriodSeconds configures graceful shutdown for backend pods. - // +optional - // +kubebuilder:validation:Minimum=0 - TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` -} - // CacheBackendConnectorStatus reports the engine-to-cache connector separately // from the optional remote L3 provider. PodLocal loopback and NodeLocal // node-derived addresses deliberately do not appear as a generic endpoint. @@ -955,12 +773,6 @@ type CacheBackendStatus struct { // +optional RemoteStorage *CacheBackendRemoteStorageStatus `json:"remoteStorage,omitempty"` - // Endpoint is the legacy remote-provider endpoint projection. New MP-aware - // clients read status.remoteStorage.endpoint; retained until repository - // consumers migrate. - // +optional - Endpoint string `json:"endpoint,omitempty"` - // MatchedEnginePods is the number of pods in this CacheBackend's namespace // whose labels match spec.engineSelector at the last reconcile. The field // is a pointer so nil ("not yet computed") is distinguishable from 0 @@ -1023,8 +835,8 @@ type CacheBackendStatus struct { // FirstAvailableAt is the stable anchor for the firstEventTimeout clock. // It latches one of two events depending on the integration mode: - // - Offload (managed): the first time the managed cache-backend - // workload was observed Available — there is a workload to wait on. + // - Offload: the first time the effective engine-side connector and any + // fail-closed remote-storage dependency were observed Ready. // - EventsOnly: the first reconcile. A server-less backend has no // workload to become Available, so it is "up" the moment it exists // and the firstEventTimeout clock starts immediately. @@ -1036,10 +848,10 @@ type CacheBackendStatus struct { // Degraded, stays Degraded until an event arrives" contract. Anchoring on // this latched value keeps the elapsed window monotonic WITHIN a serving // mode, so Degraded is sticky. It survives availability flaps and a - // recreated managed Deployment (the gate re-evaluates from the prior - // anchor, safe because a cache-server restart does not change the engine - // event source). It is NOT immortal across a mode change, though: a - // server-bearing→EventsOnly flip re-anchors it to the flip moment (and also + // recreated managed Redis Deployment (the gate re-evaluates from the prior + // anchor, safe because a Redis restart does not change the engine event + // source). It is NOT immortal across a mode change, though: an + // Offload→EventsOnly flip re-anchors it to the flip moment (and also // bypasses the sticky NoKVEventsObserved reason) so the flip gets a fresh // first-event window instead of inheriting the old mode's availability time // or timed-out verdict; and an unmanaged transition clears it so a later @@ -1049,31 +861,6 @@ type CacheBackendStatus struct { // +optional FirstAvailableAt *metav1.Time `json:"firstAvailableAt,omitempty"` - // ObservedServerInstance is the controller's cascade-decision - // baseline — a stable identifier for the Ready cache-server pod - // set the controller last anchored against. NOT a live current- - // pod-set view: the controller intentionally pins this through - // transient rolling-update midpoints and through no-Ready - // windows so the cascade does not fire on rollbacks or transient - // outages. For the live pod inventory, operators should consult - // status.matchedEnginePods (engine side) and `kubectl get pod` - // (cache-server side). - // - // Shape: `:` per Ready pod, comma-joined - // and lex-sorted by pod name. restart-sum is the per-pod - // containerStatuses[].RestartCount summed across cache-server - // containers (the names from the owned Deployment's pod - // template; foreign sidecars are excluded). Inert and cleared - // for External backends and unsupported-runtime backends. - // - // Operator-side recovery for the upstream LMCache - // LMServerConnector EPIPE-on-restart bug. See - // docs/design/cachebackend-api.md for the cascade contract, - // transition rules (which changes do / do not cascade), and - // rate-limit / no-Ready / rollback / scale-up rationale. - // +optional - ObservedServerInstance string `json:"observedServerInstance,omitempty"` - // IndexParticipation summarizes this CacheBackend's contribution to the // cluster-wide cache index — populated by the CacheIndex poller (it groups // the server's /snapshot replicas by the owning CacheBackend and projects @@ -1145,7 +932,7 @@ type CacheBackendIndexParticipation struct { // +kubebuilder:printcolumn:name="Type",type=string,JSONPath=`.spec.type` // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="Matched",type=integer,JSONPath=`.status.matchedEnginePods` -// +kubebuilder:printcolumn:name="Endpoint",type=string,JSONPath=`.status.endpoint` +// +kubebuilder:printcolumn:name="Remote",type=string,JSONPath=`.status.remoteStorage.endpoint` // +kubebuilder:printcolumn:name="Prefixes",type=integer,JSONPath=`.status.indexParticipation.prefixCount` // +kubebuilder:printcolumn:name="LastEvent",type=date,JSONPath=`.status.indexParticipation.lastEventAt` // +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` diff --git a/api/v1alpha1/cachebackend_types_test.go b/api/v1alpha1/cachebackend_types_test.go index 90f773a4..b8641fd1 100644 --- a/api/v1alpha1/cachebackend_types_test.go +++ b/api/v1alpha1/cachebackend_types_test.go @@ -10,7 +10,6 @@ import ( "reflect" "strconv" "testing" - "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -31,13 +30,9 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { "lmCache", "remoteStorage", "observation", - "deploymentKind", - "replicas", - "autoscaling", "integration", "engineSelector", "hiCache", - "template", "allowCrossNamespace", } { if !hasProperty(specSchema, field) { @@ -52,21 +47,21 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { // indexEntries was removed in #57 (it duplicated status.indexParticipation.prefixCount); // health was removed in an earlier change; capacity is removed in this PR. // All three are guarded by requireNoProperty checks below. - for _, field := range []string{"connector", "remoteStorage", "endpoint", "matchedEnginePods", "engineSelectorMessage", "failOpen", "conditions", "firstKVEventObservedAt", "firstAvailableAt"} { + for _, field := range []string{"connector", "remoteStorage", "matchedEnginePods", "engineSelectorMessage", "failOpen", "conditions", "firstKVEventObservedAt", "firstAvailableAt"} { if !hasProperty(statusSchema, field) { t.Fatalf("status.%s is missing from CRD schema", field) } } + requireNoProperty(t, statusSchema, "endpoint") + requireNoProperty(t, statusSchema, "observedServerInstance") // status.health was removed in favour of the standard // status.conditions[Ready] surface; guard against accidental // re-introduction. if hasProperty(statusSchema, "health") { t.Fatalf("status.health is present in CRD schema; it must be removed in favour of status.conditions[Ready]") } - // spec.storage.pvc + status.capacity were retired: the lm:// LMCache - // server we provision is in-memory, so a local PVC cannot back it — - // durability is a backend choice (remote store / Mooncake), not a - // generic volume knob (see docs/design/lmcache-server-persistence.md). + // Generic storage and capacity knobs remain retired: LMCache MP placement + // and optional Redis configuration own their respective memory budgets. // Guard against accidental re-introduction. if hasProperty(specSchema, "storage") { t.Fatalf("spec.storage is present in CRD schema; it was retired (durability is a backend choice — see docs/design/lmcache-server-persistence.md)") @@ -77,10 +72,6 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireEnum(t, mustProperty(t, specSchema, "type"), []string{"LMCache", "SGLangHiCache"}) requireEnum(t, mustProperty(t, specSchema, "runtime"), []string{"VLLM", "SGLang"}) - requireEnum(t, mustProperty(t, specSchema, "deploymentKind"), []string{ - "Deployment", - "StatefulSet", - }) lmCacheSchema := mustProperty(t, specSchema, "lmCache") requireNoProperty(t, lmCacheSchema, "multiprocess") requireEnum(t, mustProperty(t, lmCacheSchema, "topology"), []string{"PodLocal", "NodeLocal"}) @@ -99,18 +90,34 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxGPUWorkers"), 1) requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxCPUWorkers"), 1) requireMinimum(t, mustProperty(t, lmCacheSchema, "chunkSizeTokens"), 1) - requireMinimum(t, mustProperty(t, lmCacheSchema, "workerPort"), 1) - requireMaximum(t, mustProperty(t, lmCacheSchema, "workerPort"), 65535) remoteStorageSchema := mustProperty(t, specSchema, "remoteStorage") requireRequired(t, remoteStorageSchema, "provider") requireRequired(t, remoteStorageSchema, "ownership") - requireEnum(t, mustProperty(t, remoteStorageSchema, "provider"), []string{"Redis", "LMCacheServer", "Mooncake"}) + requireEnum(t, mustProperty(t, remoteStorageSchema, "provider"), []string{"Redis"}) requireEnum(t, mustProperty(t, remoteStorageSchema, "ownership"), []string{"Managed", "External"}) - for _, field := range []string{"endpoint", "redis", "lmCacheServer", "mooncake"} { + for _, field := range []string{"endpoint", "workload", "redis"} { if !hasProperty(remoteStorageSchema, field) { t.Fatalf("spec.remoteStorage.%s is missing from CRD schema", field) } } + workloadSchema := mustProperty(t, remoteStorageSchema, "workload") + requireNoPreserveUnknownFields(t, workloadSchema) + for _, field := range []string{ + "nodeSelector", "affinity", "tolerations", "topologySpreadConstraints", + "imagePullSecrets", "serviceAccountName", "securityContext", + "priorityClassName", "schedulerName", "runtimeClassName", + "terminationGracePeriodSeconds", + } { + if !hasProperty(workloadSchema, field) { + t.Fatalf("spec.remoteStorage.workload.%s is missing from CRD schema", field) + } + } + requireNoProperty(t, workloadSchema, "replicas") + requireNoProperty(t, workloadSchema, "autoscaling") + requireNoProperty(t, workloadSchema, "containers") + requireMinimum(t, mustProperty(t, workloadSchema, "terminationGracePeriodSeconds"), 0) + requireNoProperty(t, remoteStorageSchema, "lmCacheServer") + requireNoProperty(t, remoteStorageSchema, "mooncake") redisSchema := mustProperty(t, remoteStorageSchema, "redis") for _, field := range []string{"authentication", "tls", "database"} { if !hasProperty(redisSchema, field) { @@ -126,7 +133,7 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireMinimum(t, mustProperty(t, connectorStatusSchema, field), 0) } remoteStatusSchema := mustProperty(t, statusSchema, "remoteStorage") - requireEnum(t, mustProperty(t, remoteStatusSchema, "provider"), []string{"Redis", "LMCacheServer", "Mooncake"}) + requireEnum(t, mustProperty(t, remoteStatusSchema, "provider"), []string{"Redis"}) observationSchema := mustProperty(t, specSchema, "observation") for _, field := range []string{"modelID", "firstEventTimeout"} { if !hasProperty(observationSchema, field) { @@ -146,17 +153,7 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { if got, ok := failOpenSchema["default"].(bool); !ok || !got { t.Fatalf("integration.failOpen default = %v, want true", failOpenSchema["default"]) } - templateSchema := mustProperty(t, specSchema, "template") - requireNoPreserveUnknownFields(t, templateSchema) - for _, field := range []string{"nodeSelector", "tolerations", "affinity"} { - if !hasProperty(templateSchema, field) { - t.Fatalf("spec.template.%s is missing from CRD schema", field) - } - } - requireNoProperty(t, templateSchema, "containers") - requireNotRequired(t, specSchema, "type") - requireMinimum(t, mustProperty(t, specSchema, "replicas"), 0) hiCacheSchema := mustProperty(t, specSchema, "hiCache") requireMinimum(t, mustProperty(t, hiCacheSchema, "sizeGB"), 1) if got := mustProperty(t, hiCacheSchema, "ratio")["type"]; got != "string" { @@ -183,7 +180,6 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { if got, ok := firstEventTimeoutSchema["default"].(string); !ok || got != "5m" { t.Fatalf("observation.firstEventTimeout default = %v, want \"5m\"", firstEventTimeoutSchema["default"]) } - requireMinimum(t, mustProperty(t, templateSchema, "terminationGracePeriodSeconds"), 0) // Operator-UX defaults. Each marker below shrinks the minimum-viable // CacheBackend YAML by one field; pinning the served-schema default @@ -192,12 +188,6 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { if got, ok := mustProperty(t, specSchema, "type")["default"].(string); !ok || got != "LMCache" { t.Fatalf("spec.type default = %v, want \"LMCache\"", mustProperty(t, specSchema, "type")["default"]) } - if got, ok := mustProperty(t, specSchema, "deploymentKind")["default"].(string); !ok || got != "Deployment" { - t.Fatalf("spec.deploymentKind default = %v, want \"Deployment\"", mustProperty(t, specSchema, "deploymentKind")["default"]) - } - if got, ok := mustProperty(t, specSchema, "replicas")["default"]; !ok || !reflect.DeepEqual(got, float64(1)) { - t.Fatalf("spec.replicas default = %v (type %T), want 1", mustProperty(t, specSchema, "replicas")["default"], mustProperty(t, specSchema, "replicas")["default"]) - } requireNoProperty(t, integrationSchema, "engine") requireNoProperty(t, integrationSchema, "firstEventTimeout") if got, ok := mustProperty(t, integrationSchema, "role")["default"].(string); !ok || got != "ReadWrite" { @@ -222,13 +212,9 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { } requireMinimum(t, mustProperty(t, statusSchema, "matchedEnginePods"), 0) - // Autoscaling validation surface. - autoscalingSchema := mustProperty(t, specSchema, "autoscaling") - requireRequired(t, autoscalingSchema, "maxReplicas") - requireMinimum(t, mustProperty(t, autoscalingSchema, "minReplicas"), 1) - requireMinimum(t, mustProperty(t, autoscalingSchema, "maxReplicas"), 1) - requireMinimum(t, mustProperty(t, autoscalingSchema, "targetCPUUtilizationPercent"), 1) - requireMaximum(t, mustProperty(t, autoscalingSchema, "targetCPUUtilizationPercent"), 100) + for _, retired := range []string{"deploymentKind", "replicas", "autoscaling", "template"} { + requireNoProperty(t, specSchema, retired) + } } func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { @@ -321,14 +307,64 @@ func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { } } +func TestCacheBackendManagedWorkloadRoundTripAndDeepCopy(t *testing.T) { + runtimeClass := "gvisor" + grace := int64(45) + backend := &CacheBackend{Spec: CacheBackendSpec{ + Runtime: CacheBackendRuntimeVLLM, + RemoteStorage: &CacheBackendRemoteStorageSpec{ + Provider: CacheBackendRemoteStorageProviderRedis, + Ownership: CacheBackendRemoteStorageOwnershipManaged, + Workload: &CacheBackendManagedWorkloadSpec{ + NodeSelector: map[string]string{"pool": "cache"}, + Tolerations: []corev1.Toleration{{Key: "cache"}}, + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: func() *bool { v := true; return &v }()}, + RuntimeClassName: &runtimeClass, + TerminationGracePeriodSeconds: &grace, + }, + }, + }} + + data, err := json.Marshal(backend) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var roundTripped CacheBackend + if err := json.Unmarshal(data, &roundTripped); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !reflect.DeepEqual(backend, &roundTripped) { + t.Fatalf("JSON round trip changed object\nwant: %#v\n got: %#v", backend, &roundTripped) + } + + copied := backend.DeepCopy() + backend.Spec.RemoteStorage.Workload.NodeSelector["pool"] = "general" + backend.Spec.RemoteStorage.Workload.Tolerations[0].Key = "general" + *backend.Spec.RemoteStorage.Workload.SecurityContext.RunAsNonRoot = false + *backend.Spec.RemoteStorage.Workload.RuntimeClassName = "kata" + *backend.Spec.RemoteStorage.Workload.TerminationGracePeriodSeconds = 60 + + workload := copied.Spec.RemoteStorage.Workload + if workload.NodeSelector["pool"] != "cache" || workload.Tolerations[0].Key != "cache" { + t.Fatalf("managed workload scheduling was not deep-copied") + } + if workload.SecurityContext == nil || workload.SecurityContext.RunAsNonRoot == nil || !*workload.SecurityContext.RunAsNonRoot { + t.Fatalf("managed workload securityContext was not deep-copied") + } + if workload.RuntimeClassName == nil || *workload.RuntimeClassName != "gvisor" || + workload.TerminationGracePeriodSeconds == nil || *workload.TerminationGracePeriodSeconds != 45 { + t.Fatalf("managed workload runtime/grace fields were not deep-copied: %+v", workload) + } +} + func TestCacheBackendCRDPrintColumns(t *testing.T) { version := loadCacheBackendCRDVersion(t, "v1alpha1") columns := mustPath[[]any](t, version, "additionalPrinterColumns") want := map[string]string{ - "Ready": `.status.conditions[?(@.type=="Ready")].status`, - "Endpoint": ".status.endpoint", - "Matched": ".status.matchedEnginePods", + "Ready": `.status.conditions[?(@.type=="Ready")].status`, + "Remote": ".status.remoteStorage.endpoint", + "Matched": ".status.matchedEnginePods", } seen := map[string]string{} for _, column := range columns { @@ -351,230 +387,6 @@ func TestCacheBackendCRDPrintColumns(t *testing.T) { } } -func TestCacheBackendDeepCopyCopiesNestedFields(t *testing.T) { - replicas := int32(2) - hitRate := "0.50" - t2HitRate := "0.66" - matchedEnginePods := int32(7) - firstKVEventAt := metav1.NewTime(time.Unix(1_700_000_000, 0).UTC()) - firstAvailableAt := metav1.NewTime(time.Unix(1_700_000_500, 0).UTC()) - runAsNonRoot := true - runtimeClassName := "runc" - terminationGracePeriodSeconds := int64(30) - autoscalingMin := int32(2) - autoscalingTargetCPU := int32(70) - hiCacheSize := int32(64) - chunkSize := int32(128) - workerPort := int32(5555) - hostCapacity := resource.MustParse("6Gi") - providerMemory := resource.MustParse("2Gi") - observationTimeout := metav1.Duration{Duration: 3 * time.Minute} - backend := &CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "default"}, - Spec: CacheBackendSpec{ - Runtime: CacheBackendRuntimeSGLang, - Type: CacheBackendTypeLMCache, - DeploymentKind: CacheBackendDeploymentKindStatefulSet, - LMCache: &LMCacheEngineSpec{ - ChunkSizeTokens: &chunkSize, - HostMemory: &CacheBackendHostMemorySpec{Capacity: &hostCapacity}, - WorkerPort: &workerPort, - }, - RemoteStorage: &CacheBackendRemoteStorageSpec{ - Provider: CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &LMCacheServerRemoteStorageSpec{ - Command: []string{"lmcache_server", "--flag"}, - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{corev1.ResourceMemory: providerMemory}, - }, - }, - }, - Observation: &CacheBackendObservationSpec{ - ModelID: "model-a", - FirstEventTimeout: &observationTimeout, - }, - Replicas: &replicas, - Autoscaling: &CacheBackendAutoscalingSpec{ - MinReplicas: &autoscalingMin, - MaxReplicas: 5, - TargetCPUUtilizationPercent: &autoscalingTargetCPU, - }, - Integration: &CacheBackendIntegrationSpec{ - Role: CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &CacheBackendEngineSelector{ - MatchLabels: map[string]string{"inferencecache.io/cache-enabled": "true"}, - }, - HiCache: &SGLangHiCacheSpec{ - SizeGB: &hiCacheSize, - WritePolicy: SGLangHiCacheWriteThrough, - }, - Template: &CacheBackendPodSpecOverride{ - NodeSelector: map[string]string{"pool": "cache"}, - Tolerations: []corev1.Toleration{{ - Key: "cache", - Operator: corev1.TolerationOpExists, - }}, - SecurityContext: &corev1.PodSecurityContext{ - RunAsNonRoot: &runAsNonRoot, - }, - RuntimeClassName: &runtimeClassName, - TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - }, - }, - Status: CacheBackendStatus{ - Endpoint: "cache.default.svc:8080", - IndexParticipation: &CacheBackendIndexParticipation{ - PrefixCount: 7, - HitRate: &hitRate, - T2HitRate: &t2HitRate, - }, - MatchedEnginePods: &matchedEnginePods, - EngineSelectorMessage: "spec.engineSelector.matchLabels={app:engine}; no Pods in namespace match", - FirstKVEventObservedAt: &firstKVEventAt, - FirstAvailableAt: &firstAvailableAt, - Conditions: []metav1.Condition{{ - Type: "Ready", - Status: metav1.ConditionTrue, - Reason: "Available", - Message: "backend is ready", - LastTransitionTime: metav1.Now(), - }}, - }, - } - - copied := backend.DeepCopy() - *backend.Spec.Replicas = 3 - *backend.Spec.Autoscaling.MinReplicas = 4 - backend.Spec.Autoscaling.MaxReplicas = 9 - *backend.Spec.Autoscaling.TargetCPUUtilizationPercent = 90 - *backend.Spec.LMCache.ChunkSizeTokens = 256 - changedHostCapacity := resource.MustParse("12Gi") - *backend.Spec.LMCache.HostMemory.Capacity = changedHostCapacity - *backend.Spec.LMCache.WorkerPort = 6666 - backend.Spec.RemoteStorage.LMCacheServer.Command[0] = "changed" - backend.Spec.RemoteStorage.LMCacheServer.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") - backend.Spec.Observation.ModelID = "changed" - backend.Spec.Observation.FirstEventTimeout.Duration = time.Hour - *backend.Spec.HiCache.SizeGB = 128 - backend.Spec.EngineSelector.MatchLabels["inferencecache.io/cache-enabled"] = "false" - backend.Spec.Template.NodeSelector["pool"] = "general" - backend.Spec.Template.Tolerations[0].Key = "general" - backend.Status.IndexParticipation.PrefixCount = 99 - *backend.Status.IndexParticipation.HitRate = "0.99" - *backend.Spec.Template.SecurityContext.RunAsNonRoot = false - *backend.Spec.Template.RuntimeClassName = "kata" - *backend.Spec.Template.TerminationGracePeriodSeconds = 60 - *backend.Status.MatchedEnginePods = 11 - backend.Status.EngineSelectorMessage = "changed" - *backend.Status.FirstKVEventObservedAt = metav1.NewTime(time.Unix(0, 0).UTC()) - *backend.Status.FirstAvailableAt = metav1.NewTime(time.Unix(0, 0).UTC()) - backend.Status.Conditions[0].Message = "changed" - - if copied.Spec.Replicas == nil || *copied.Spec.Replicas != 2 { - t.Fatalf("replicas was not deep-copied") - } - if copied.Spec.Autoscaling == nil { - t.Fatalf("autoscaling was not deep-copied") - } - if copied.Spec.Autoscaling.MinReplicas == nil || *copied.Spec.Autoscaling.MinReplicas != 2 { - t.Fatalf("autoscaling.minReplicas was not deep-copied") - } - if copied.Spec.Autoscaling.MaxReplicas != 5 { - t.Fatalf("autoscaling.maxReplicas was not deep-copied") - } - if copied.Spec.Autoscaling.TargetCPUUtilizationPercent == nil || *copied.Spec.Autoscaling.TargetCPUUtilizationPercent != 70 { - t.Fatalf("autoscaling.targetCPUUtilizationPercent was not deep-copied") - } - if copied.Spec.LMCache == nil || - copied.Spec.LMCache.ChunkSizeTokens == nil || - *copied.Spec.LMCache.ChunkSizeTokens != 128 || - copied.Spec.LMCache.HostMemory == nil || - copied.Spec.LMCache.HostMemory.Capacity == nil || - copied.Spec.LMCache.HostMemory.Capacity.Cmp(resource.MustParse("6Gi")) != 0 || - copied.Spec.LMCache.WorkerPort == nil || - *copied.Spec.LMCache.WorkerPort != 5555 { - t.Fatalf("lmCache nested fields were not deep-copied") - } - if copied.Spec.RemoteStorage == nil || - copied.Spec.RemoteStorage.LMCacheServer == nil || - copied.Spec.RemoteStorage.LMCacheServer.Command[0] != "lmcache_server" || - copied.Spec.RemoteStorage.LMCacheServer.Resources == nil { - t.Fatalf("remoteStorage.lmCacheServer nested fields were not deep-copied") - } - copiedProviderMemory := copied.Spec.RemoteStorage.LMCacheServer.Resources.Limits[corev1.ResourceMemory] - if copiedProviderMemory.Cmp(resource.MustParse("2Gi")) != 0 { - t.Fatalf("remoteStorage.lmCacheServer resources were not deep-copied") - } - if copied.Spec.Observation == nil || - copied.Spec.Observation.ModelID != "model-a" || - copied.Spec.Observation.FirstEventTimeout == nil || - copied.Spec.Observation.FirstEventTimeout.Duration != 3*time.Minute { - t.Fatalf("observation nested fields were not deep-copied") - } - if copied.Spec.Integration == nil { - t.Fatalf("integration was not deep-copied") - } - if copied.Spec.HiCache == nil || copied.Spec.HiCache.SizeGB == nil || *copied.Spec.HiCache.SizeGB != 64 { - t.Fatalf("hiCache.sizeGB was not deep-copied") - } - if copied.Spec.EngineSelector == nil { - t.Fatalf("engineSelector was not deep-copied") - } - if copied.Spec.EngineSelector.MatchLabels["inferencecache.io/cache-enabled"] != "true" { - t.Fatalf("engineSelector.matchLabels was not deep-copied") - } - if copied.Spec.Template == nil { - t.Fatalf("template was not deep-copied") - } - if copied.Spec.Template.NodeSelector["pool"] != "cache" { - t.Fatalf("template.nodeSelector was not deep-copied") - } - if copied.Spec.Template.Tolerations[0].Key != "cache" { - t.Fatalf("template.tolerations was not deep-copied") - } - if copied.Spec.Template.SecurityContext == nil || - copied.Spec.Template.SecurityContext.RunAsNonRoot == nil || - !*copied.Spec.Template.SecurityContext.RunAsNonRoot { - t.Fatalf("template.securityContext was not deep-copied") - } - if copied.Spec.Template.RuntimeClassName == nil || *copied.Spec.Template.RuntimeClassName != "runc" { - t.Fatalf("template.runtimeClassName was not deep-copied") - } - if copied.Spec.Template.TerminationGracePeriodSeconds == nil || - *copied.Spec.Template.TerminationGracePeriodSeconds != 30 { - t.Fatalf("template.terminationGracePeriodSeconds was not deep-copied") - } - if copied.Status.IndexParticipation == nil || - copied.Status.IndexParticipation.PrefixCount != 7 { - t.Fatalf("status.indexParticipation.prefixCount was not deep-copied") - } - if copied.Status.IndexParticipation.HitRate == nil || - *copied.Status.IndexParticipation.HitRate != "0.50" { - t.Fatalf("status.indexParticipation.hitRate was not deep-copied") - } - if copied.Status.IndexParticipation.T2HitRate == nil || - *copied.Status.IndexParticipation.T2HitRate != "0.66" { - t.Fatalf("status.indexParticipation.t2HitRate was not deep-copied") - } - if copied.Status.MatchedEnginePods == nil || *copied.Status.MatchedEnginePods != 7 { - t.Fatalf("status.matchedEnginePods was not deep-copied") - } - if copied.Status.EngineSelectorMessage != "spec.engineSelector.matchLabels={app:engine}; no Pods in namespace match" { - t.Fatalf("status.engineSelectorMessage was not deep-copied") - } - if copied.Status.FirstKVEventObservedAt == nil || !copied.Status.FirstKVEventObservedAt.Time.Equal(time.Unix(1_700_000_000, 0).UTC()) { - t.Fatalf("status.firstKVEventObservedAt was not deep-copied") - } - if copied.Status.FirstAvailableAt == nil || !copied.Status.FirstAvailableAt.Time.Equal(time.Unix(1_700_000_500, 0).UTC()) { - t.Fatalf("status.firstAvailableAt was not deep-copied") - } - if copied.Status.Conditions[0].Message != "backend is ready" { - t.Fatalf("conditions were not deep-copied") - } -} - func TestCacheBackendJSONOmitEmptySpecPointers(t *testing.T) { data, err := json.Marshal(CacheBackendSpec{}) if err != nil { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a18cf5d9..5de0b0b8 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -43,31 +43,6 @@ func (in *CacheBackend) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheBackendAutoscalingSpec) DeepCopyInto(out *CacheBackendAutoscalingSpec) { - *out = *in - if in.MinReplicas != nil { - in, out := &in.MinReplicas, &out.MinReplicas - *out = new(int32) - **out = **in - } - if in.TargetCPUUtilizationPercent != nil { - in, out := &in.TargetCPUUtilizationPercent, &out.TargetCPUUtilizationPercent - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendAutoscalingSpec. -func (in *CacheBackendAutoscalingSpec) DeepCopy() *CacheBackendAutoscalingSpec { - if in == nil { - return nil - } - out := new(CacheBackendAutoscalingSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendConnectorStatus) DeepCopyInto(out *CacheBackendConnectorStatus) { *out = *in @@ -105,26 +80,6 @@ func (in *CacheBackendEngineSelector) DeepCopy() *CacheBackendEngineSelector { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheBackendHostMemorySpec) DeepCopyInto(out *CacheBackendHostMemorySpec) { - *out = *in - if in.Capacity != nil { - in, out := &in.Capacity, &out.Capacity - x := (*in).DeepCopy() - *out = &x - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendHostMemorySpec. -func (in *CacheBackendHostMemorySpec) DeepCopy() *CacheBackendHostMemorySpec { - if in == nil { - return nil - } - out := new(CacheBackendHostMemorySpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendIndexParticipation) DeepCopyInto(out *CacheBackendIndexParticipation) { *out = *in @@ -212,27 +167,7 @@ func (in *CacheBackendList) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheBackendObservationSpec) DeepCopyInto(out *CacheBackendObservationSpec) { - *out = *in - if in.FirstEventTimeout != nil { - in, out := &in.FirstEventTimeout, &out.FirstEventTimeout - *out = new(metav1.Duration) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendObservationSpec. -func (in *CacheBackendObservationSpec) DeepCopy() *CacheBackendObservationSpec { - if in == nil { - return nil - } - out := new(CacheBackendObservationSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CacheBackendPodSpecOverride) DeepCopyInto(out *CacheBackendPodSpecOverride) { +func (in *CacheBackendManagedWorkloadSpec) DeepCopyInto(out *CacheBackendManagedWorkloadSpec) { *out = *in if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector @@ -282,12 +217,32 @@ func (in *CacheBackendPodSpecOverride) DeepCopyInto(out *CacheBackendPodSpecOver } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendPodSpecOverride. -func (in *CacheBackendPodSpecOverride) DeepCopy() *CacheBackendPodSpecOverride { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendManagedWorkloadSpec. +func (in *CacheBackendManagedWorkloadSpec) DeepCopy() *CacheBackendManagedWorkloadSpec { if in == nil { return nil } - out := new(CacheBackendPodSpecOverride) + out := new(CacheBackendManagedWorkloadSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheBackendObservationSpec) DeepCopyInto(out *CacheBackendObservationSpec) { + *out = *in + if in.FirstEventTimeout != nil { + in, out := &in.FirstEventTimeout, &out.FirstEventTimeout + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendObservationSpec. +func (in *CacheBackendObservationSpec) DeepCopy() *CacheBackendObservationSpec { + if in == nil { + return nil + } + out := new(CacheBackendObservationSpec) in.DeepCopyInto(out) return out } @@ -295,21 +250,16 @@ func (in *CacheBackendPodSpecOverride) DeepCopy() *CacheBackendPodSpecOverride { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendRemoteStorageSpec) DeepCopyInto(out *CacheBackendRemoteStorageSpec) { *out = *in + if in.Workload != nil { + in, out := &in.Workload, &out.Workload + *out = new(CacheBackendManagedWorkloadSpec) + (*in).DeepCopyInto(*out) + } if in.Redis != nil { in, out := &in.Redis, &out.Redis *out = new(RedisRemoteStorageSpec) (*in).DeepCopyInto(*out) } - if in.LMCacheServer != nil { - in, out := &in.LMCacheServer, &out.LMCacheServer - *out = new(LMCacheServerRemoteStorageSpec) - (*in).DeepCopyInto(*out) - } - if in.Mooncake != nil { - in, out := &in.Mooncake, &out.Mooncake - *out = new(MooncakeRemoteStorageSpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendRemoteStorageSpec. @@ -355,16 +305,6 @@ func (in *CacheBackendSpec) DeepCopyInto(out *CacheBackendSpec) { *out = new(CacheBackendObservationSpec) (*in).DeepCopyInto(*out) } - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - if in.Autoscaling != nil { - in, out := &in.Autoscaling, &out.Autoscaling - *out = new(CacheBackendAutoscalingSpec) - (*in).DeepCopyInto(*out) - } if in.Integration != nil { in, out := &in.Integration, &out.Integration *out = new(CacheBackendIntegrationSpec) @@ -380,11 +320,6 @@ func (in *CacheBackendSpec) DeepCopyInto(out *CacheBackendSpec) { *out = new(SGLangHiCacheSpec) (*in).DeepCopyInto(*out) } - if in.Template != nil { - in, out := &in.Template, &out.Template - *out = new(CacheBackendPodSpecOverride) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendSpec. @@ -919,16 +854,6 @@ func (in *LMCacheEngineSpec) DeepCopyInto(out *LMCacheEngineSpec) { *out = new(int32) **out = **in } - if in.HostMemory != nil { - in, out := &in.HostMemory, &out.HostMemory - *out = new(CacheBackendHostMemorySpec) - (*in).DeepCopyInto(*out) - } - if in.WorkerPort != nil { - in, out := &in.WorkerPort, &out.WorkerPort - *out = new(int32) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheEngineSpec. @@ -1054,56 +979,6 @@ func (in *LMCachePodLocalSpec) DeepCopy() *LMCachePodLocalSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LMCacheServerRemoteStorageSpec) DeepCopyInto(out *LMCacheServerRemoteStorageSpec) { - *out = *in - if in.Command != nil { - in, out := &in.Command, &out.Command - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheServerRemoteStorageSpec. -func (in *LMCacheServerRemoteStorageSpec) DeepCopy() *LMCacheServerRemoteStorageSpec { - if in == nil { - return nil - } - out := new(LMCacheServerRemoteStorageSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MooncakeRemoteStorageSpec) DeepCopyInto(out *MooncakeRemoteStorageSpec) { - *out = *in - if in.Command != nil { - in, out := &in.Command, &out.Command - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Resources != nil { - in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MooncakeRemoteStorageSpec. -func (in *MooncakeRemoteStorageSpec) DeepCopy() *MooncakeRemoteStorageSpec { - if in == nil { - return nil - } - out := new(MooncakeRemoteStorageSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PDAcceleratorTypeSpec) DeepCopyInto(out *PDAcceleratorTypeSpec) { *out = *in diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 731ad9e2..2a682bc0 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -53,7 +53,6 @@ type options struct { cacheIndexRefreshEvery time.Duration policyPushEvery time.Duration subscriberImage string - lmCacheServerImage string policyServerGRPCAddress string zapOpts zap.Options } @@ -70,7 +69,6 @@ func defaultOptions() options { cacheIndexRefreshEvery: controller.DefaultRefreshInterval, policyPushEvery: controller.DefaultPolicyPushInterval, subscriberImage: "", - lmCacheServerImage: "", policyServerGRPCAddress: "inference-cache-server.inference-cache-system.svc.cluster.local:9090", zapOpts: zap.Options{ TimeEncoder: zapcore.RFC3339TimeEncoder, @@ -91,7 +89,6 @@ func parseOptions() options { flag.DurationVar(&opts.cacheIndexRefreshEvery, "cacheindex-refresh-interval", opts.cacheIndexRefreshEvery, "How often to refresh the CacheIndex status from the server snapshot.") flag.DurationVar(&opts.policyPushEvery, "cachepolicy-push-interval", opts.policyPushEvery, "How often to re-push the full CachePolicy snapshot to the server (self-healing on server restart).") flag.StringVar(&opts.subscriberImage, "kvevent-subscriber-image", opts.subscriberImage, "Image reference the pod-mutating webhook uses for the kvevent-subscriber sidecar it auto-attaches to managed-LMCache engine pods (vLLM and SGLang). Empty (default) disables auto-attach — the engine pod wiring still happens but no subscriber container is appended. Pin to a digest in production.") - flag.StringVar(&opts.lmCacheServerImage, "lmcache-server-image", opts.lmCacheServerImage, "Fallback image for managed LMCache servers when spec.remoteStorage.lmCacheServer.image is empty. Managed LMCache reconciliation requires one of these settings. Pin to a client-compatible digest in production.") flag.StringVar(&opts.policyServerGRPCAddress, "policy-server-grpc-address", opts.policyServerGRPCAddress, "host:port the kvevent-subscriber sidecar dials to ReportCacheState. Defaults to the in-cluster Service DNS in the inference-cache-system namespace.") opts.zapOpts.BindFlags(flag.CommandLine) flag.Parse() @@ -131,14 +128,13 @@ func main() { // webhooks. Whatever admission accepts is therefore renderable and // injectable without each caller remembering extra registrations. // - // The kvevent-subscriber sidecar image, managed LMCache server fallback, - // and policy-server gRPC address are operator-supplied. Pinning images to - // compatible digests in production and selecting the right Service DNS are + // The kvevent-subscriber sidecar image and policy-server gRPC address are + // operator-supplied. Pinning images to compatible digests in production and + // selecting the right Service DNS are // deployment concerns; a CacheBackend may still override its own provider // image when needed. adapterRegistries := builtinadapters.New(builtinadapters.Options{ SubscriberImage: opts.subscriberImage, - LMCacheServerImage: opts.lmCacheServerImage, PolicyServerGRPCAddress: opts.policyServerGRPCAddress, }) adapterRegistry := adapterRegistries.Runtime @@ -219,7 +215,7 @@ func main() { // The Pod admission handler uses the manager's APIReader (uncached // live client) instead of the cached client: pod CREATE is a // one-shot opportunity to inject, so a stale informer view of the - // owning CacheBackend (in particular a status.endpoint that lags + // owning CacheBackend (in particular remote-storage status that lags // reality) would leave the pod permanently unwired. Live reads also // avoid a cold-cache window on controller startup. mgr.GetWebhookServer().Register(podwebhook.WebhookPath, &webhook.Admission{ diff --git a/cmd/inferencecache/doctor.go b/cmd/inferencecache/doctor.go index c9de3561..58527bd0 100644 --- a/cmd/inferencecache/doctor.go +++ b/cmd/inferencecache/doctor.go @@ -279,8 +279,8 @@ func readToken(path string) string { return strings.TrimSpace(string(b)) } -// dialTCP reports whether a TCP connection to addr succeeds. addr may carry an -// lm:// scheme (LMCache endpoints) which is stripped before dialing. +// dialTCP reports whether a TCP connection to addr succeeds. HTTP URL schemes +// are stripped before dialing. func dialTCP(ctx context.Context, addr string) error { addr = stripScheme(addr) var d net.Dialer @@ -292,7 +292,7 @@ func dialTCP(ctx context.Context, addr string) error { } func stripScheme(addr string) string { - for _, scheme := range []string{"lm://", "http://", "https://"} { + for _, scheme := range []string{"http://", "https://"} { if strings.HasPrefix(addr, scheme) { return strings.TrimPrefix(addr, scheme) } diff --git a/cmd/inferencecache/doctor_unit_test.go b/cmd/inferencecache/doctor_unit_test.go index 0c961bc2..4b51bf78 100644 --- a/cmd/inferencecache/doctor_unit_test.go +++ b/cmd/inferencecache/doctor_unit_test.go @@ -28,7 +28,6 @@ func svc(ns, name string) *corev1.Service { func TestStripScheme(t *testing.T) { cases := map[string]string{ - "lm://host:1": "host:1", "http://host:2": "host:2", "https://host:3": "host:3", "host:4": "host:4", diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 437b86ee..2d99b86d 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -26,8 +26,8 @@ spec: - jsonPath: .status.matchedEnginePods name: Matched type: integer - - jsonPath: .status.endpoint - name: Endpoint + - jsonPath: .status.remoteStorage.endpoint + name: Remote type: string - jsonPath: .status.indexParticipation.prefixCount name: Prefixes @@ -61,11 +61,7 @@ spec: metadata: type: object spec: - description: |- - CacheBackendSpec defines the desired state of a cache backend. - - The autoscaling spec (spec.autoscaling) is reconciled into a - HorizontalPodAutoscaler for managed backends. + description: CacheBackendSpec defines the desired state of a cache backend. properties: allowCrossNamespace: description: |- @@ -76,61 +72,6 @@ spec: the cluster operator should explicitly acknowledge. Endpoints that are not in-cluster Service DNS (external hostnames, IPs) are unaffected. type: boolean - autoscaling: - description: |- - Autoscaling configures horizontal autoscaling for the managed backend - workload. When set, the controller reconciles a HorizontalPodAutoscaler - owned by this CacheBackend; the HPA then drives the underlying workload's - replica count, overriding spec.replicas. - properties: - maxReplicas: - description: MaxReplicas is the upper bound for the HPA replica - count. - format: int32 - minimum: 1 - type: integer - minReplicas: - description: |- - MinReplicas is the lower bound for the HPA replica count. The - admission defaulter computes the default at write time from - spec.replicas (which itself defaults to 1) so the HPA's floor matches - the operator-declared baseline rather than a hard-coded constant. This - is a FIRST-APPLY-ONLY default: the defaulter never overwrites an - operator-set value, AND once stamped the field is owned by the - apiserver field manager — subsequent edits to spec.replicas do NOT - recompute or move minReplicas, matching the standard Kubernetes HPA - convention that scaling intent flows through HPA fields once an HPA - owns the workload. To widen or narrow the autoscaling band post-apply, - edit spec.autoscaling.minReplicas directly. Operators who want a - non-default floor on first apply set the field explicitly. - format: int32 - minimum: 1 - type: integer - targetCPUUtilizationPercent: - description: |- - TargetCPUUtilizationPercent is the average per-pod CPU utilization the - HPA targets. Defaults to 80 when unset. - format: int32 - maximum: 100 - minimum: 1 - type: integer - required: - - maxReplicas - type: object - x-kubernetes-validations: - - message: minReplicas must not exceed maxReplicas - rule: '!has(self.minReplicas) || self.minReplicas <= self.maxReplicas' - deploymentKind: - default: Deployment - description: |- - DeploymentKind identifies whether a managed backend is reconciled as a - Deployment or StatefulSet. Defaults to Deployment — the only kind the - Phase-1 reconciler templates; StatefulSet is reserved for future - per-replica-PVC topologies and is a no-op today. - enum: - - Deployment - - StatefulSet - type: string engineSelector: description: |- EngineSelector selects which engine pods this CacheBackend claims via @@ -139,18 +80,17 @@ spec: metav1.LabelSelector surface (matchExpressions, operator-based selection) is NOT exposed today — only MatchLabels. Pods that match get runtime-adapter engine wiring injected by the - mutating Pod admission webhook at pod CREATE time. Server-backed - adapters require status.endpoint to be published first; the webhook - fail-opens when it is empty. Engine-local adapters such as native - SGLang HiCache, and the LMCache host-only path, explicitly require no - endpoint and inject immediately. + mutating Pod admission webhook at pod CREATE time. LMCache MP adapters + inject the local connector immediately; when an optional managed Redis + tier is configured, its address is read from status.remoteStorage.endpoint. + Engine-local adapters such as native SGLang HiCache require no endpoint. Admission is CREATE-only; recovery or a configuration update requires recreating the pod (e.g. `kubectl rollout restart`), not editing its live labels. This describes the default spec.integration.mode=Offload path. For spec.integration.mode=EventsOnly no KV connector wiring (env vars + - CLI args) is injected and status.endpoint is neither required nor + CLI args) is injected and no connector or remote-storage endpoint is published — the kvevent-subscriber observation sidecar alone is injected (see below), so matched pods report cache state for routing without offloading KV to a backend server. @@ -232,29 +172,6 @@ spec: description: Integration describes how inference engines should use the cache backend. properties: - engineHostNetwork: - description: |- - EngineHostNetwork opts engine pods bound to this backend into host - networking. It exists for exactly one backend today: Mooncake, whose - transfer engine is a peer-to-peer mesh — the engine dials a real node IP - on a dynamically negotiated port, which a CNI overlay pod IP cannot do. - Without this the backend reconciles Ready and transfers zero KV, and - admission warns as much on every apply. - - This is opt-in, and deliberately not a default, because it rewrites the - networking of a pod the operator owns: - - hostNetwork is a privilege. A Pod Security "restricted" namespace - rejects such a pod, and because mutating webhooks run BEFORE Pod - Security validation, silently injecting it would turn a working engine - pod into a rejected one — with an error that names Pod Security, not - this controller. - - The pod's ports move onto the node's interfaces, outside the pod - network. NetworkPolicy selects pods by pod IP and therefore stops - constraining them (see docs/design/cachebackend-api.md). - - Admission rejects this on any backend type that does not need it, so it - can never sit inert on a CacheBackend. - type: boolean engineOverrides: description: |- EngineOverrides lets the operator amend the non-reserved args / env @@ -511,8 +428,8 @@ spec: EventsOnly is the supported integration for hybrid-attention models that cannot take a vLLM KV connector (and a lighter routing-only deployment for anyone who does not want an offload tier). Because EventsOnly provisions - no server, status.endpoint stays empty and the autoscaling spec is - rejected at admission; Ready is still gated on the first observed KV event. + no server, connector and remote-storage status stay empty; Ready is still + gated on the first observed KV event. SGLangHiCache supports Offload only and is rejected with EventsOnly. See the CacheBackendIntegrationMode godoc. enum: @@ -549,23 +466,6 @@ spec: format: int32 minimum: 1 type: integer - hostMemory: - description: |- - HostMemory is a legacy flat-field input retained only while repository - consumers migrate to podLocal.server.l1Capacity. - properties: - capacity: - anyOf: - - type: integer - - type: string - description: Capacity is the memory budget for the engine-side - host cache. - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - x-kubernetes-validations: - - message: capacity must be greater than zero - rule: quantity(string(self)).isGreaterThan(quantity('0')) - type: object nodeLocal: description: |- NodeLocal configures a future per-node MP server. Admission currently @@ -1765,33 +1665,16 @@ spec: required: - server type: object - remoteSerde: - description: RemoteSerde is a legacy in-process input and is forbidden - with MP. - type: string topology: description: |- Topology selects the canonical LMCache MP server placement. PodLocal is implemented first; NodeLocal is reserved and rejected until Phase 8. - Omit this field only for a legacy in-process/flat-field object during the - repository migration window. enum: - PodLocal - NodeLocal type: string - workerImage: - description: |- - WorkerImage is a legacy flat-field input retained only while repository - consumers migrate to podLocal.server.image. - type: string - workerPort: - description: |- - WorkerPort is a legacy flat-field input retained only while repository - consumers migrate to podLocal.server.port. - format: int32 - maximum: 65535 - minimum: 1 - type: integer + required: + - topology type: object observation: description: |- @@ -1818,157 +1701,6 @@ spec: Endpoint is required for External ownership and rejected for Managed ownership, whose endpoint is controller-observed. type: string - lmCacheServer: - description: LMCacheServer contains standalone lmcache-server-owned - configuration. - properties: - command: - description: Command overrides the managed server command - and arguments. - items: - minLength: 1 - type: string - minItems: 1 - type: array - image: - description: Image is used only when ownership is Managed. - type: string - resources: - description: Resources are applied to the managed lmcache-server - container. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - type: object - mooncake: - description: Mooncake contains Mooncake-owned configuration. - properties: - command: - description: Command overrides the managed Mooncake master - command and arguments. - items: - minLength: 1 - type: string - minItems: 1 - type: array - image: - description: Image is used only when ownership is Managed. - type: string - resources: - description: Resources are applied to the managed Mooncake - master container. - properties: - claims: - description: |- - Claims lists the names of resources, defined in spec.resourceClaims, - that are used by this container. - - This field depends on the - DynamicResourceAllocation feature gate. - - This field is immutable. It can only be set for containers. - items: - description: ResourceClaim references one entry in PodSpec.ResourceClaims. - properties: - name: - description: |- - Name must match the name of one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes that resource available - inside a container. - type: string - request: - description: |- - Request is the name chosen for a request in the referenced claim. - If empty, everything from the claim is made available, otherwise - only the result of this request. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Limits describes the maximum amount of compute resources allowed. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - Requests describes the minimum amount of compute resources required. - If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, - otherwise to an implementation-defined value. Requests cannot exceed Limits. - More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - type: object - type: object - type: object ownership: description: |- Ownership identifies whether inference-cache manages the provider @@ -1981,8 +1713,6 @@ spec: description: Provider identifies the remote-storage technology. enum: - Redis - - LMCacheServer - - Mooncake type: string redis: description: Redis contains Redis-owned configuration. @@ -2147,277 +1877,430 @@ spec: - caCertificate type: object type: object - required: - - ownership - - provider - type: object - replicas: - default: 1 - description: |- - Replicas is the desired number of backend workload replicas. Defaults - to 1 — a conservative single-replica deployment; operators opt into - horizontal scale via spec.autoscaling. - - When spec.autoscaling is set, the HPA owns the live replica count and - the autoscaling floor (spec.autoscaling.minReplicas) is auto-defaulted - to spec.replicas on FIRST APPLY ONLY by the admission defaulter. - Subsequent edits to spec.replicas do NOT move the autoscaling floor — - minReplicas is operator-owned (and operator-pinned via the apiserver - field manager) after first apply, matching the standard Kubernetes HPA - convention that scaling intent flows through HPA fields once an HPA - owns the workload. To widen or narrow the autoscaling band post-apply, - edit spec.autoscaling.minReplicas directly. - format: int32 - minimum: 0 - type: integer - runtime: - description: |- - Runtime identifies the inference runtime. Values are case-sensitive: use - VLLM or SGLang. - enum: - - VLLM - - SGLang - type: string - template: - description: Template provides pod-level overrides for managed backend - workloads. - properties: - affinity: - description: Affinity configures backend pod scheduling affinity. + workload: + description: |- + Workload configures Pod scheduling and security for controller-managed + provider workloads. It is rejected when ownership is External. properties: - nodeAffinity: - description: Describes node affinity scheduling rules for - the pod. + affinity: + description: Affinity configures provider Pod scheduling affinity. properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated with - the corresponding weight. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching the - corresponding nodeSelectorTerm, in the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. + nodeAffinity: + description: Describes node affinity scheduling rules + for the pod. properties: - nodeSelectorTerms: - description: Required. A list of node selector terms. - The terms are ORed. + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. items: description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector requirements - by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector requirements - by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that the selector - applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic - type: object - podAffinity: - description: Describes pod affinity scheduling rules (e.g. - co-locate this pod in the same node, zone, etc. as some - other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. + preference: + description: A node selector term, associated + with the corresponding weight. properties: matchExpressions: - description: matchExpressions is a list - of label selector requirements. The requirements - are ANDed. + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, in the + range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: |- + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: + description: |- + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. items: description: |- A label selector requirement is a selector that contains values, a key, and an operator that @@ -2499,273 +2382,275 @@ spec: of label selector requirements. The requirements are ANDed. items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key - that the selector applies to. - type: string - operator: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + x-kubernetes-list-type: atomic + topologyKey: + description: |- + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + x-kubernetes-list-type: atomic + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the anti-affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity expressions, etc.), + compute a sum by iterating through the elements of this field and subtracting + "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the + node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, + associated with the corresponding weight. + properties: + labelSelector: + description: |- + A label query over a set of resources, in this case pods. + If it's null, this PodAffinityTerm matches with no Pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both matchLabelKeys and labelSelector. + Also, matchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + mismatchLabelKeys: + description: |- + MismatchLabelKeys is a set of pod label keys to select which pods will + be taken into consideration. The keys are used to lookup values from the + incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` + to select the group of existing pods which pods will be taken into consideration + for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming + pod labels will be ignored. The default value is empty. + The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. + Also, mismatchLabelKeys cannot be set when labelSelector isn't set. + items: + type: string + type: array + x-kubernetes-list-type: atomic + namespaceSelector: + description: |- + A label query over the set of namespaces that the term applies to. + The term is applied to the union of the namespaces selected by this field + and the ones listed in the namespaces field. + null selector and null or empty namespaces list means "this pod's namespace". + An empty selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: |- + namespaces specifies a static list of namespace names that the term applies to. + The term is applied to the union of the namespaces listed in this field + and the ones selected by namespaceSelector. + null or empty namespaces list and null namespaceSelector means "this pod's namespace". + items: + type: string type: array x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + topologyKey: description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching + the labelSelector in the specified namespaces, where co-located is defined as running on a node + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + weight: description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string + weight associated with matching the corresponding podAffinityTerm, + in the range 1-100. + format: int32 + type: integer required: - - topologyKey + - podAffinityTerm + - weight type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the anti-affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the anti-affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a pod label update), the + system may or may not try to eventually evict the pod from its node. + When there are multiple elements, the lists of nodes corresponding to each + podAffinityTerm are intersected, i.e. all terms must be satisfied. + items: description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic - type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling rules - (e.g. avoid putting this pod in the same node, zone, etc. - as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched WeightedPodAffinityTerm - fields are added per-node to find the most preferred - node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, associated - with the corresponding weight. + Defines a set of pods (namely those matching the labelSelector + relative to the given namespace(s)) that this pod should be + co-located (affinity) or not co-located (anti-affinity) with, + where co-located is defined as running on a node whose value of + the label with key matches that of any node on which + a pod of the set of pods is running properties: labelSelector: description: |- @@ -2910,695 +2795,529 @@ spec: description: |- This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + whose value of the label with key topologyKey matches that of any node on which any of the + selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic + type: array + x-kubernetes-list-type: atomic + type: object type: object - type: object - imagePullSecrets: - description: ImagePullSecrets references secrets used to pull - backend pod images. - items: - description: |- - LocalObjectReference contains enough information to let you locate the - referenced object inside the same namespace. - properties: - name: - default: "" + imagePullSecrets: + description: ImagePullSecrets references Secrets used to pull + provider images. + items: description: |- - Name of the referent. - This field is effectively required, but due to backwards compatibility is - allowed to be empty. Instances of this type with an empty value here are - almost certainly wrong. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + nodeSelector: + additionalProperties: type: string - type: object - x-kubernetes-map-type: atomic - type: array - nodeSelector: - additionalProperties: - type: string - description: NodeSelector constrains backend pods to nodes with - matching labels. - type: object - priorityClassName: - description: PriorityClassName is the priority class assigned - to backend pods. - type: string - runtimeClassName: - description: RuntimeClassName selects the runtime class used for - backend pods. - type: string - schedulerName: - description: SchedulerName selects the scheduler used for backend - pods. - type: string - securityContext: - description: SecurityContext configures pod-level security settings - for backend pods. - properties: - appArmorProfile: - description: |- - appArmorProfile is the AppArmor options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. + description: NodeSelector constrains provider Pods to nodes + with matching labels. + type: object + priorityClassName: + description: PriorityClassName is the priority class assigned + to provider Pods. + type: string + runtimeClassName: + description: RuntimeClassName selects the runtime class used + for provider Pods. + type: string + schedulerName: + description: SchedulerName selects the scheduler used for + provider Pods. + type: string + securityContext: + description: SecurityContext configures Pod-level security + settings for provider Pods. properties: - localhostProfile: + appArmorProfile: description: |- - localhostProfile indicates a profile loaded on the node that should be used. - The profile must be preconfigured on the node to work. - Must match the loaded name of the profile. - Must be set if and only if type is "Localhost". - type: string - type: + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: + description: |- + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type + type: object + fsGroup: description: |- - type indicates which kind of AppArmor profile will be applied. - Valid options are: - Localhost - a profile pre-loaded on the node. - RuntimeDefault - the container runtime's default profile. - Unconfined - no AppArmor enforcement. - type: string - required: - - type - type: object - fsGroup: - description: |- - A special supplemental group that applies to all containers in a pod. - Some volume types allow the Kubelet to change the ownership of that volume - to be owned by the pod: + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: - 1. The owning GID will be the FSGroup - 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) - 3. The permission bits are OR'd with rw-rw---- + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- - If unset, the Kubelet will not modify the ownership and permissions of any volume. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - fsGroupChangePolicy: - description: |- - fsGroupChangePolicy defines behavior of changing ownership and permission of the volume - before being exposed inside Pod. This field will only apply to - volume types which support fsGroup based ownership(and permissions). - It will have no effect on ephemeral volume types such as: secret, configmaps - and emptydir. - Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. - Note that this field cannot be set when spec.os.name is windows. - type: string - runAsGroup: - description: |- - The GID to run the entrypoint of the container process. - Uses runtime default if unset. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - runAsNonRoot: - description: |- - Indicates that the container must run as a non-root user. - If true, the Kubelet will validate the image at runtime to ensure that it - does not run as UID 0 (root) and fail to start the container if it does. - If unset or false, no such validation will be performed. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: boolean - runAsUser: - description: |- - The UID to run the entrypoint of the container process. - Defaults to user specified in image metadata if unspecified. - May also be set in SecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence - for that container. - Note that this field cannot be set when spec.os.name is windows. - format: int64 - type: integer - seLinuxChangePolicy: - description: |- - seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. - It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. - Valid values are "MountOption" and "Recursive". + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". - "Recursive" means relabeling of all files on all Pod volumes by the container runtime. - This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. - "MountOption" mounts all eligible Pod volumes with `-o context` mount option. - This requires all Pods that share the same volume to use the same SELinux label. - It is not possible to share the same volume among privileged and unprivileged Pods. - Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes - whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their - CSIDriver instance. Other volumes are always re-labelled recursively. - "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. - If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. - If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes - and "Recursive" for all other volumes. + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. - This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. - All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. - Note that this field cannot be set when spec.os.name is windows. - type: string - seLinuxOptions: - description: |- - The SELinux context to be applied to all containers. - If unspecified, the container runtime will allocate a random SELinux context for each - container. May also be set in SecurityContext. If set in - both SecurityContext and PodSecurityContext, the value specified in SecurityContext - takes precedence for that container. - Note that this field cannot be set when spec.os.name is windows. - properties: - level: - description: Level is SELinux level label that applies - to the container. - type: string - role: - description: Role is a SELinux role label that applies - to the container. - type: string - type: - description: Type is a SELinux type label that applies - to the container. - type: string - user: - description: User is a SELinux user label that applies - to the container. + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. type: string - type: object - seccompProfile: - description: |- - The seccomp options to use by the containers in this pod. - Note that this field cannot be set when spec.os.name is windows. - properties: - localhostProfile: + seLinuxOptions: description: |- - localhostProfile indicates a profile defined in a file on the node should be used. - The profile must be preconfigured on the node to work. - Must be a descending path, relative to the kubelet's configured seccomp profile location. - Must be set if type is "Localhost". Must NOT be set for any other type. - type: string - type: + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + seccompProfile: description: |- - type indicates which kind of seccomp profile will be applied. - Valid options are: + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. + properties: + localhostProfile: + description: |- + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: + description: |- + type indicates which kind of seccomp profile will be applied. + Valid options are: - Localhost - a profile defined in a file on the node should be used. - RuntimeDefault - the container runtime default profile should be used. - Unconfined - no profile should be applied. + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. type: string - required: - - type + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to be + set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of + the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string + type: object type: object - supplementalGroups: - description: |- - A list of groups applied to the first process run in each container, in - addition to the container's primary GID and fsGroup (if specified). If - the SupplementalGroupsPolicy feature is enabled, the - supplementalGroupsPolicy field determines whether these are in addition - to or instead of any group memberships defined in the container image. - If unspecified, no additional groups are added, though group memberships - defined in the container image may still be used, depending on the - supplementalGroupsPolicy field. - Note that this field cannot be set when spec.os.name is windows. - items: - format: int64 - type: integer - type: array - x-kubernetes-list-type: atomic - supplementalGroupsPolicy: - description: |- - Defines how supplemental groups of the first container processes are calculated. - Valid values are "Merge" and "Strict". If not specified, "Merge" is used. - (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled - and the container runtime must implement support for this feature. - Note that this field cannot be set when spec.os.name is windows. + serviceAccountName: + description: ServiceAccountName is the ServiceAccount used + by provider Pods. type: string - sysctls: - description: |- - Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported - sysctls (by the container runtime) might fail to launch. - Note that this field cannot be set when spec.os.name is windows. + terminationGracePeriodSeconds: + description: TerminationGracePeriodSeconds configures graceful + provider shutdown. + format: int64 + minimum: 0 + type: integer + tolerations: + description: Tolerations allow provider Pods to schedule onto + tainted nodes. items: - description: Sysctl defines a kernel parameter to be set + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . properties: - name: - description: Name of a property to set + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer value: - description: Value of a property to set + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. type: string - required: - - name - - value type: object type: array - x-kubernetes-list-type: atomic - windowsOptions: + topologySpreadConstraints: description: |- - The Windows specific settings applied to all containers. - If unspecified, the options within a container's SecurityContext will be used. - If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. - Note that this field cannot be set when spec.os.name is linux. - properties: - gmsaCredentialSpec: - description: |- - GMSACredentialSpec is where the GMSA admission webhook - (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the - GMSA credential spec named by the GMSACredentialSpecName field. - type: string - gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the - GMSA credential spec to use. - type: string - hostProcess: - description: |- - HostProcess determines if a container should be run as a 'Host Process' container. - All of a Pod's containers must have the same effective HostProcess value - (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). - In addition, if HostProcess is true then HostNetwork must also be set to true. - type: boolean - runAsUserName: - description: |- - The UserName in Windows to run the entrypoint of the container process. - Defaults to the user specified in image metadata if unspecified. - May also be set in PodSecurityContext. If set in both SecurityContext and - PodSecurityContext, the value specified in SecurityContext takes precedence. - type: string - type: object - type: object - serviceAccountName: - description: ServiceAccountName is the service account used by - backend pods. - type: string - terminationGracePeriodSeconds: - description: TerminationGracePeriodSeconds configures graceful - shutdown for backend pods. - format: int64 - minimum: 0 - type: integer - tolerations: - description: Tolerations allow backend pods to schedule onto tainted - nodes. - items: - description: |- - The pod this Toleration is attached to tolerates any taint that matches - the triple using the matching operator . - properties: - effect: - description: |- - Effect indicates the taint effect to match. Empty means match all taint effects. - When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: |- - Key is the taint key that the toleration applies to. Empty means match all taint keys. - If the key is empty, operator must be Exists; this combination means to match all values and all keys. - type: string - operator: - description: |- - Operator represents a key's relationship to the value. - Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. - Exists is equivalent to wildcard for value, so that a pod can - tolerate all taints of a particular category. - Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). - type: string - tolerationSeconds: - description: |- - TolerationSeconds represents the period of time the toleration (which must be - of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, - it is not set, which means tolerate the taint forever (do not evict). Zero and - negative values will be treated as 0 (evict immediately) by the system. - format: int64 - type: integer - value: - description: |- - Value is the taint value the toleration matches to. - If the operator is Exists, the value should be empty, otherwise just a regular string. - type: string - type: object - type: array - topologySpreadConstraints: - description: TopologySpreadConstraints configures backend pod - spreading across topology domains. - items: - description: TopologySpreadConstraint specifies how to spread - matching pods among the given topology. - properties: - labelSelector: - description: |- - LabelSelector is used to find matching pods. - Pods that match this label selector are counted to determine the number of pods - in their corresponding topology domain. + TopologySpreadConstraints configures provider Pod spreading across + topology domains. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: + labelSelector: + description: |- + LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine the number of pods + in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + description: |- + MatchLabelKeys is a set of pod label keys to select the pods over which + spreading will be calculated. The keys are used to lookup values from the + incoming pod labels, those key-value labels are ANDed with labelSelector + to select the group of existing pods over which spreading will be calculated + for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. + MatchLabelKeys cannot be set when LabelSelector isn't set. + Keys that don't exist in the incoming pod labels will + be ignored. A null or empty list means only match against labelSelector. + + This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). + items: + type: string type: array x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string + maxSkew: description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select the pods over which - spreading will be calculated. The keys are used to lookup values from the - incoming pod labels, those key-value labels are ANDed with labelSelector - to select the group of existing pods over which spreading will be calculated - for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. - MatchLabelKeys cannot be set when LabelSelector isn't set. - Keys that don't exist in the incoming pod labels will - be ignored. A null or empty list means only match against labelSelector. - - This is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default). - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxSkew: - description: |- - MaxSkew describes the degree to which pods may be unevenly distributed. - When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference - between the number of matching pods in the target topology and the global minimum. - The global minimum is the minimum number of matching pods in an eligible domain - or zero if the number of eligible domains is less than MinDomains. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 2/2/1: - In this case, the global minimum is 1. - | zone1 | zone2 | zone3 | - | P P | P P | P | - - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; - scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) - violate MaxSkew(1). - - if MaxSkew is 2, incoming pod can be scheduled onto any zone. - When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence - to topologies that satisfy it. - It's a required field. Default value is 1 and 0 is not allowed. - format: int32 - type: integer - minDomains: - description: |- - MinDomains indicates a minimum number of eligible domains. - When the number of eligible domains with matching topology keys is less than minDomains, - Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. - And when the number of eligible domains with matching topology keys equals or greater than minDomains, - this value has no effect on scheduling. - As a result, when the number of eligible domains is less than minDomains, - scheduler won't schedule more than maxSkew Pods to those domains. - If value is nil, the constraint behaves as if MinDomains is equal to 1. - Valid values are integers greater than 0. - When value is not nil, WhenUnsatisfiable must be DoNotSchedule. + MaxSkew describes the degree to which pods may be unevenly distributed. + When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference + between the number of matching pods in the target topology and the global minimum. + The global minimum is the minimum number of matching pods in an eligible domain + or zero if the number of eligible domains is less than MinDomains. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 2/2/1: + In this case, the global minimum is 1. + | zone1 | zone2 | zone3 | + | P P | P P | P | + - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; + scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) + violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto any zone. + When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence + to topologies that satisfy it. + It's a required field. Default value is 1 and 0 is not allowed. + format: int32 + type: integer + minDomains: + description: |- + MinDomains indicates a minimum number of eligible domains. + When the number of eligible domains with matching topology keys is less than minDomains, + Pod Topology Spread treats "global minimum" as 0, and then the calculation of Skew is performed. + And when the number of eligible domains with matching topology keys equals or greater than minDomains, + this value has no effect on scheduling. + As a result, when the number of eligible domains is less than minDomains, + scheduler won't schedule more than maxSkew Pods to those domains. + If value is nil, the constraint behaves as if MinDomains is equal to 1. + Valid values are integers greater than 0. + When value is not nil, WhenUnsatisfiable must be DoNotSchedule. - For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same - labelSelector spread as 2/2/2: - | zone1 | zone2 | zone3 | - | P P | P P | P P | - The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. - In this situation, new pod with the same labelSelector cannot be scheduled, - because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, - it will violate MaxSkew. - format: int32 - type: integer - nodeAffinityPolicy: - description: |- - NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector - when calculating pod topology spread skew. Options are: - - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. + For example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same + labelSelector spread as 2/2/2: + | zone1 | zone2 | zone3 | + | P P | P P | P P | + The number of domains is less than 5(MinDomains), so "global minimum" is treated as 0. + In this situation, new pod with the same labelSelector cannot be scheduled, + because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, + it will violate MaxSkew. + format: int32 + type: integer + nodeAffinityPolicy: + description: |- + NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector + when calculating pod topology spread skew. Options are: + - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. + - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations. - If this value is nil, the behavior is equivalent to the Honor policy. - type: string - nodeTaintsPolicy: - description: |- - NodeTaintsPolicy indicates how we will treat node taints when calculating - pod topology spread skew. Options are: - - Honor: nodes without taints, along with tainted nodes for which the incoming pod - has a toleration, are included. - - Ignore: node taints are ignored. All nodes are included. + If this value is nil, the behavior is equivalent to the Honor policy. + type: string + nodeTaintsPolicy: + description: |- + NodeTaintsPolicy indicates how we will treat node taints when calculating + pod topology spread skew. Options are: + - Honor: nodes without taints, along with tainted nodes for which the incoming pod + has a toleration, are included. + - Ignore: node taints are ignored. All nodes are included. - If this value is nil, the behavior is equivalent to the Ignore policy. - type: string - topologyKey: - description: |- - TopologyKey is the key of node labels. Nodes that have a label with this key - and identical values are considered to be in the same topology. - We consider each as a "bucket", and try to put balanced number - of pods into each bucket. - We define a domain as a particular instance of a topology. - Also, we define an eligible domain as a domain whose nodes meet the requirements of - nodeAffinityPolicy and nodeTaintsPolicy. - e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. - And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. - It's a required field. - type: string - whenUnsatisfiable: - description: |- - WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy - the spread constraint. - - DoNotSchedule (default) tells the scheduler not to schedule it. - - ScheduleAnyway tells the scheduler to schedule the pod in any location, - but giving higher precedence to topologies that would help reduce the - skew. - A constraint is considered "Unsatisfiable" for an incoming pod - if and only if every possible node assignment for that pod would violate - "MaxSkew" on some topology. - For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same - labelSelector spread as 3/1/1: - | zone1 | zone2 | zone3 | - | P P P | P | P | - If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled - to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies - MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler - won't make it *more* imbalanced. - It's a required field. - type: string - required: - - maxSkew - - topologyKey - - whenUnsatisfiable - type: object - type: array + If this value is nil, the behavior is equivalent to the Ignore policy. + type: string + topologyKey: + description: |- + TopologyKey is the key of node labels. Nodes that have a label with this key + and identical values are considered to be in the same topology. + We consider each as a "bucket", and try to put balanced number + of pods into each bucket. + We define a domain as a particular instance of a topology. + Also, we define an eligible domain as a domain whose nodes meet the requirements of + nodeAffinityPolicy and nodeTaintsPolicy. + e.g. If TopologyKey is "kubernetes.io/hostname", each Node is a domain of that topology. + And, if TopologyKey is "topology.kubernetes.io/zone", each zone is a domain of that topology. + It's a required field. + type: string + whenUnsatisfiable: + description: |- + WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy + the spread constraint. + - DoNotSchedule (default) tells the scheduler not to schedule it. + - ScheduleAnyway tells the scheduler to schedule the pod in any location, + but giving higher precedence to topologies that would help reduce the + skew. + A constraint is considered "Unsatisfiable" for an incoming pod + if and only if every possible node assignment for that pod would violate + "MaxSkew" on some topology. + For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: + | zone1 | zone2 | zone3 | + | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled + to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler + won't make it *more* imbalanced. + It's a required field. + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + type: object + required: + - ownership + - provider type: object + runtime: + description: |- + Runtime identifies the inference runtime. Values are case-sensitive: use + VLLM or SGLang. + enum: + - VLLM + - SGLang + type: string type: default: LMCache description: |- @@ -3733,12 +3452,6 @@ spec: minimum: 0 type: integer type: object - endpoint: - description: |- - Endpoint is the legacy remote-provider endpoint projection. New MP-aware - clients read status.remoteStorage.endpoint; retained until repository - consumers migrate. - type: string engineSelectorMessage: description: |- EngineSelectorMessage explains the current engineSelector matching @@ -3760,8 +3473,8 @@ spec: description: |- FirstAvailableAt is the stable anchor for the firstEventTimeout clock. It latches one of two events depending on the integration mode: - - Offload (managed): the first time the managed cache-backend - workload was observed Available — there is a workload to wait on. + - Offload: the first time the effective engine-side connector and any + fail-closed remote-storage dependency were observed Ready. - EventsOnly: the first reconcile. A server-less backend has no workload to become Available, so it is "up" the moment it exists and the firstEventTimeout clock starts immediately. @@ -3773,10 +3486,10 @@ spec: Degraded, stays Degraded until an event arrives" contract. Anchoring on this latched value keeps the elapsed window monotonic WITHIN a serving mode, so Degraded is sticky. It survives availability flaps and a - recreated managed Deployment (the gate re-evaluates from the prior - anchor, safe because a cache-server restart does not change the engine - event source). It is NOT immortal across a mode change, though: a - server-bearing→EventsOnly flip re-anchors it to the flip moment (and also + recreated managed Redis Deployment (the gate re-evaluates from the prior + anchor, safe because a Redis restart does not change the engine event + source). It is NOT immortal across a mode change, though: an + Offload→EventsOnly flip re-anchors it to the flip moment (and also bypasses the sticky NoKVEventsObserved reason) so the flip gets a fresh first-event window instead of inheriting the old mode's availability time or timed-out verdict; and an unmanaged transition clears it so a later @@ -3884,31 +3597,6 @@ spec: format: int64 minimum: 0 type: integer - observedServerInstance: - description: |- - ObservedServerInstance is the controller's cascade-decision - baseline — a stable identifier for the Ready cache-server pod - set the controller last anchored against. NOT a live current- - pod-set view: the controller intentionally pins this through - transient rolling-update midpoints and through no-Ready - windows so the cascade does not fire on rollbacks or transient - outages. For the live pod inventory, operators should consult - status.matchedEnginePods (engine side) and `kubectl get pod` - (cache-server side). - - Shape: `:` per Ready pod, comma-joined - and lex-sorted by pod name. restart-sum is the per-pod - containerStatuses[].RestartCount summed across cache-server - containers (the names from the owned Deployment's pod - template; foreign sidecars are excluded). Inert and cleared - for External backends and unsupported-runtime backends. - - Operator-side recovery for the upstream LMCache - LMServerConnector EPIPE-on-restart bug. See - docs/design/cachebackend-api.md for the cascade contract, - transition rules (which changes do / do not cascade), and - rate-limit / no-Ready / rollback / scale-up rationale. - type: string remoteStorage: description: |- RemoteStorage reports the optional remote L3 independently from the @@ -3922,8 +3610,6 @@ spec: optional shared/remote cache tier. enum: - Redis - - LMCacheServer - - Mooncake type: string ready: description: |- diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 00832b43..6ea18663 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -25,7 +25,6 @@ spec: containers: - args: - --leader-elect - - --lmcache-server-image=lmcache/standalone:v0.4.7 command: - /controller image: controller:latest diff --git a/config/observability/kustomization.yaml b/config/observability/kustomization.yaml index 94523595..7727708c 100644 --- a/config/observability/kustomization.yaml +++ b/config/observability/kustomization.yaml @@ -26,8 +26,8 @@ # pods: `inference-cache-server.inference-cache-system.svc.cluster.local:8080` # (server-side series — index, lookup, auth) AND the # `inference-cache-controller-manager` pod's `:8080` (controller-side -# series — per-stage probe-result counter, cache-server restart-cascade -# counter) plus every injected PodLocal LMCache sidecar's `:8080/metrics`. +# per-stage probe-result counter) plus every injected PodLocal LMCache +# sidecar's `:8080/metrics`. # The controller-side alerts (ServerProbeFail today) read # controller-emitted series, so a server-only scrape leaves them inert. # diff --git a/config/observability/podmonitor.yaml b/config/observability/podmonitor.yaml index f0814fa0..db407986 100644 --- a/config/observability/podmonitor.yaml +++ b/config/observability/podmonitor.yaml @@ -5,11 +5,9 @@ # PodMonitor for `inference-cache-controller-manager`. Without this, a # prometheus-operator install scrapes ONLY the server-binary metrics # (covered by servicemonitor.yaml) and silently misses every -# controller-binary series — including +# controller-binary series, including # `inferencecache_backend_probe_result_total` (which `ServerProbeFail` -# in this same overlay depends on) and -# `inferencecache_backend_server_restart_cascades_total` (no alert -# reads it today; reserved for a future cascade-loop alert). +# in this same overlay depends on). # # A PodMonitor (not a ServiceMonitor) is the right shape here because the # controller binary does not front its /metrics endpoint with a Service — diff --git a/config/overlays/gpu-validation/kustomization.yaml b/config/overlays/gpu-validation/kustomization.yaml index 4e5be615..969be60d 100644 --- a/config/overlays/gpu-validation/kustomization.yaml +++ b/config/overlays/gpu-validation/kustomization.yaml @@ -25,9 +25,6 @@ patches: kind: Deployment name: inference-cache-controller-manager patch: | - - op: replace - path: /spec/template/spec/containers/0/args/1 - value: --lmcache-server-image=lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 - op: add path: /spec/template/spec/containers/0/args/- value: --kvevent-subscriber-image=sjc.ocir.io/idqj093njucb/inference-cache-subscriber@sha256:2fdaa611642a0f2c48b6c7a7257ff28d75030e4bf6df3cebb589319f2e48e504 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d3520e3d..88b51572 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -51,18 +51,6 @@ rules: - replicasets verbs: - get -- apiGroups: - - autoscaling - resources: - - horizontalpodautoscalers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - inferencecache.io resources: diff --git a/config/samples/_test/cachebackend-invalid-scale-to-zero-no-min.yaml b/config/samples/_test/cachebackend-invalid-scale-to-zero-no-min.yaml deleted file mode 100644 index 60a0d9fe..00000000 --- a/config/samples/_test/cachebackend-invalid-scale-to-zero-no-min.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# verify-samples: skip -# -# INTENTIONALLY INVALID fixture — NOT a shippable sample. -# -# This CacheBackend pairs spec.replicas=0 with an enabled spec.autoscaling that -# leaves spec.autoscaling.minReplicas unset. The CacheBackend validating -# admission webhook MUST reject this shape: with replicas=0 the defaulter -# declines to stamp minReplicas (a 0 value would violate the schema's -# Minimum=1), the apiserver would otherwise accept the CR with minReplicas -# unset, and the reconciler's HPA fallback silently picks minReplicas=1 — so an -# operator who wrote "scale to zero" gets "scale 1-N" with no notification. -# -# The default-install smoke gate applies this file and asserts the live -# admission webhook rejects it with the locked wording AND that the CR is not -# persisted. It is deliberately parked under config/samples/_test/ with the -# `# verify-samples: skip` opt-out above so neither `make verify-samples` nor -# the install-smoke apply-clean backstop treats it as a shippable sample. -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: cachebackend-invalid-scale-to-zero-no-min -spec: - runtime: VLLM - type: LMCache - replicas: 0 - autoscaling: - maxReplicas: 5 diff --git a/config/samples/cachebackend-cpu-override.yaml b/config/samples/cachebackend-cpu-override.yaml index 2b782dbc..60923110 100644 --- a/config/samples/cachebackend-cpu-override.yaml +++ b/config/samples/cachebackend-cpu-override.yaml @@ -22,8 +22,6 @@ metadata: spec: runtime: VLLM type: LMCache - deploymentKind: Deployment - replicas: 1 integration: role: ReadWrite # Per-engine overrides applied AFTER the runtime adapter produces its diff --git a/config/samples/cachebackend-events-only.yaml b/config/samples/cachebackend-events-only.yaml index e3b3c32e..64cbde63 100644 --- a/config/samples/cachebackend-events-only.yaml +++ b/config/samples/cachebackend-events-only.yaml @@ -8,9 +8,9 @@ # # Set spec.integration.mode: EventsOnly to select it. Compared to the default # Offload mode (see cachebackend-lmcache.yaml), the controller here: -# - provisions NO standalone lmcache-server Deployment or Service, and leaves -# status.endpoint empty (there is no server address to publish), and -# - injects NO `--kv-transfer-config` arg / LMCACHE_* env into the engine +# - provisions NO remote-storage Deployment or Service, and leaves +# connector/remote-storage status empty (there is no server address), and +# - injects NO KV-offload connector or MP server into the engine Pod # container. # The mutating Pod webhook appends the kvevent-subscriber sidecar to matched # engine pods — but only when the controller runs with --kvevent-subscriber-image @@ -34,12 +34,12 @@ # it sits Ready=False/AwaitingFirstKVEvent until the auto-attached # kvevent-subscriber reports an event (requires the controller to run with # --kvevent-subscriber-image set; empty by default), then flips to -# Ready=True/KVEventsObserved. status.endpoint stays empty throughout. +# Ready=True/KVEventsObserved. Connector and remote-storage status stay empty. # # EventsOnly takes precedence over host-tier/offload configuration. Omit # spec.lmCache and spec.remoteStorage: no LMCache host tier is configured, and # admission rejects a remote provider because no connector would dial it. -# Autoscaling is likewise rejected because there is no server workload. +# There is no backend workload to scale in this mode. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -61,5 +61,4 @@ spec: # keys per-replica entries by model. A hybrid-attention model is the # canonical events-only use case. modelID: Qwen/Qwen3-Next-80B-A3B-Instruct - # No autoscaling: there is no server workload to scale (admission rejects it). - # No remoteStorage: events-only publishes no status.endpoint. + # No remoteStorage: events-only has no shared cache tier. diff --git a/config/samples/cachebackend-external.yaml b/config/samples/cachebackend-external.yaml index 0ae047e6..03050ec3 100644 --- a/config/samples/cachebackend-external.yaml +++ b/config/samples/cachebackend-external.yaml @@ -8,8 +8,8 @@ # trusted private network: LMCache 0.5.3 authentication is supported, but TLS is # not, and admission rejects a TLS block rather than silently ignoring it. # -# This sample explicitly chooses Redis. A legacy external LMCacheServer endpoint -# cannot be converted automatically because replacing lm:// changes sharing and +# This sample explicitly chooses Redis. A removed external LMCacheServer endpoint +# cannot be converted automatically because replacing its IP wire changes sharing and # data-plane semantics. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend diff --git a/config/samples/cachebackend-lmcache.yaml b/config/samples/cachebackend-lmcache.yaml index 3126a93a..053041d4 100644 --- a/config/samples/cachebackend-lmcache.yaml +++ b/config/samples/cachebackend-lmcache.yaml @@ -7,7 +7,7 @@ # matching engine Pod and points it at Redis; it never replaces the # inference-owner's engine image. This sample explicitly selects Redis to keep # cross-Pod sharing. It is not an automatic mapping from the legacy -# LMCacheServer provider, whose lm:// data plane had different semantics. +# removed LMCacheServer provider, whose IP data plane had different semantics. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -43,6 +43,11 @@ spec: remoteStorage: provider: Redis ownership: Managed + # Optional scheduling/security applies only to the controller-managed + # provider Pod. It never mutates the inference-engine Pod. + workload: + nodeSelector: + kubernetes.io/os: linux redis: image: docker.io/library/redis@sha256:e7723ff73d963f5cc6d9c4643ea3d989527a402a319239054e9472a7fb9219a2 resources: diff --git a/config/samples/cachebackend-with-override.yaml b/config/samples/cachebackend-with-override.yaml index e055583c..2b2b35b7 100644 --- a/config/samples/cachebackend-with-override.yaml +++ b/config/samples/cachebackend-with-override.yaml @@ -31,8 +31,6 @@ metadata: spec: runtime: VLLM type: LMCache - deploymentKind: Deployment - replicas: 1 integration: role: ReadWrite # Amend the engine container the mutating Pod webhook injects. diff --git a/config/samples/recipe-cpu-dev.yaml b/config/samples/recipe-cpu-dev.yaml index 18da47e6..dd38aaab 100644 --- a/config/samples/recipe-cpu-dev.yaml +++ b/config/samples/recipe-cpu-dev.yaml @@ -56,7 +56,6 @@ metadata: spec: runtime: VLLM type: LMCache - replicas: 1 integration: role: ReadWrite engineSelector: diff --git a/config/samples/recipe-external-cache.yaml b/config/samples/recipe-external-cache.yaml index 7b370901..1b43daf9 100644 --- a/config/samples/recipe-external-cache.yaml +++ b/config/samples/recipe-external-cache.yaml @@ -5,7 +5,7 @@ # Recipe: External Redis L3 — point the MP server at Redis you manage. # # Scenario: you already run Redis and do not want the controller to provision -# it. External ownership skips the remote-provider Deployment/Service/HPA. The +# it. External ownership skips the remote-provider Deployment and Service. The # pod webhook still injects one PodLocal LMCache MP server per engine Pod, and # that server connects to the explicit RESP endpoint. # diff --git a/config/samples/recipe-gpu-production.yaml b/config/samples/recipe-gpu-production.yaml index c9689b07..9bbcfbaf 100644 --- a/config/samples/recipe-gpu-production.yaml +++ b/config/samples/recipe-gpu-production.yaml @@ -15,7 +15,7 @@ # * The engine Deployment requests a GPU (`nvidia.com/gpu: 1`) and uses the # CUDA vLLM image — schedule it onto GPU nodes. # * Redis is intentionally a singleton soft-state L3; this recipe does not -# map the legacy LMCacheServer autoscaling shape because independent Redis +# map the removed LMCacheServer scaling shape because independent Redis # replicas would partition the keyspace rather than preserve semantics. # * A CachePolicy tunes eviction for the namespace (see the CachePolicy doc # pointer in config/samples/README.md). diff --git a/config/samples/recipe-multi-tenant.yaml b/config/samples/recipe-multi-tenant.yaml index 550aa74c..9f345537 100644 --- a/config/samples/recipe-multi-tenant.yaml +++ b/config/samples/recipe-multi-tenant.yaml @@ -89,7 +89,6 @@ metadata: spec: runtime: VLLM type: LMCache - replicas: 1 integration: role: ReadWrite engineSelector: @@ -122,7 +121,6 @@ metadata: spec: runtime: VLLM type: LMCache - replicas: 1 integration: role: ReadWrite engineSelector: diff --git a/config/samples/recipe-tuning.yaml b/config/samples/recipe-tuning.yaml index be632cf4..9322399f 100644 --- a/config/samples/recipe-tuning.yaml +++ b/config/samples/recipe-tuning.yaml @@ -36,7 +36,6 @@ metadata: spec: runtime: VLLM type: LMCache - replicas: 1 integration: role: ReadWrite engineOverrides: diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 99ee0247..7a6abbfd 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -33,7 +33,7 @@ CacheBackend / engine-pod data path, then tenant and policy configuration: | 2 | `/snapshot` reachability | HTTP GET returns 200 with a JSON-parseable body (bearer token if available; flags the unauthenticated path) | | 3 | `/policy` reachability | the route is wired (non-mutating HEAD; 2xx/401/403/405 = wired, 404 = not mounted, 5xx = WARN) | | 4 | `/probe` reachability | the controller-driven functional self-test route (same `:8081` listener + auth profile) is wired (non-mutating HEAD; same status classification as `/policy`) | -| 5 | Per-CacheBackend health | `Ready=True`; managed backends have ever observed a KV event (durable `firstKVEventObservedAt` latch) and, if `lastEventAt` is present, it is fresh (drained backends with a cleared `lastEventAt` are not flagged); `FunctionalProbeOK` is not failing (when the condition is present, a non-`True` value surfaces *why* Ready is downgraded); `status.endpoint` populated and reachable | +| 5 | Per-CacheBackend health | `Ready=True`; managed backends have ever observed a KV event (durable `firstKVEventObservedAt` latch) and, if `lastEventAt` is present, it is fresh (drained backends with a cleared `lastEventAt` are not flagged); `FunctionalProbeOK` is not failing; an optional `status.remoteStorage.endpoint` is reachable | | 6 | Engine-pod injection audit | every pod matching a CacheBackend `engineSelector` carries the `inferencecache.io/injected-by` annotation (or the injection Event) | | 7 | Orphan-pod check | pods with a `NoMatchingCacheBackend` Event in the last 24h (forward-looking — no producer yet, see Notes) | | 8 | CacheTenant health | `QuotaExceeded` condition is not `True` | @@ -65,7 +65,7 @@ Every finding carries a stable, greppable code. Codes are permanent identifiers | `CB002` | WARN | managed backend with a selector matches 0 engine pods (LikelySelectorMismatch) | | `CB003` | WARN | no KV event ever observed for the backend (EngineNotReportingState) | | `CB004` | WARN | last KV event is stale (EngineStale) | -| `CB005` | WARN | `status.endpoint` empty or unreachable | +| `CB005` | WARN | Configured remote-storage endpoint is empty or unreachable | | `CB006` | OK | CacheBackend healthy on every applicable axis | | `CB007` | WARN | `FunctionalProbeOK` condition present but not `True` — the controller's functional self-test is failing for this backend (explains a Ready downgrade) | | `EP001` | WARN | matched engine pod missing an injection marker (no `inferencecache.io/injected-by` annotation and no Event) | @@ -173,11 +173,11 @@ The declarative Kubernetes-config checks (per-CacheBackend health, injection audit, orphan-pod, CacheTenant, CachePolicy — minus the per-CacheBackend TCP dial) work from anywhere your kubeconfig can reach the apiserver. The live endpoint probes (server gRPC health, `/snapshot`, `/policy`, `/probe`) and the -per-CacheBackend `status.endpoint` TCP dial need network reachability to the +per-CacheBackend remote-storage endpoint TCP dial needs network reachability to the cache-plane server's gRPC `:9090` / snapshot-policy-probe `:8081` ports and to each backend's endpoint — which from a workstation are usually in-cluster Service DNS / ClusterIPs that do not resolve. `--config-only` skips the endpoint -probes and the per-CacheBackend TCP dial (it still validates `status.endpoint` +probes and the per-CacheBackend TCP dial (it still validates the remote endpoint is published), so it is the right mode from a workstation without a port-forward: - **In-cluster**: the server is discovered by Service DNS. Note the two diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index bd4f0325..e47f2aa6 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -4,9 +4,11 @@ Status: implemented · Tracks: InferenceCache tech spec §4.1 · API group: `inf > **Current production contract:** LMCache uses typed > `spec.lmCache.topology: PodLocal` multiprocess wiring for both vLLM and -> SGLang, with optional Redis selected explicitly. Topology-less LMCacheServer, -> Mooncake, `lm://`, and IP-connector descriptions below are retained only as -> legacy compatibility/history until Phase 7; they are not recommended paths. +> SGLang, with optional Redis selected explicitly. References to topology-less +> LMCacheServer, the former IP-wired Mooncake provider, `lm://`, and the IP +> connector are explicitly marked history for behavior physically removed in +> Phase 7. Mooncake remains a planned typed MP L2 provider; it is not available +> in the current API and is never translated to Redis. `CacheBackend` is the namespaced CRD that describes an engine-side cache implementation, an optional remote-storage tier, and the engine integration @@ -52,6 +54,10 @@ spec: remoteStorage: provider: Redis ownership: Managed + workload: + nodeSelector: + cache-tier: shared + serviceAccountName: cache-provider redis: image: docker.io/library/redis:7.4-alpine resources: @@ -66,7 +72,9 @@ spec: - `lmCache` and `hiCache` configure local/host cache behavior. - `remoteStorage.provider` selects the optional remote technology. - `remoteStorage.ownership` selects controller-managed or external lifecycle. -- Provider-specific workload settings live below their provider object. +- Generic managed-workload scheduling and Pod security live under + `remoteStorage.workload`; provider image, resources, and topology remain + provider-specific. - `observation` owns event-observation identity and timing. Omitting `remoteStorage` is meaningful and never selects infrastructure: @@ -103,19 +111,17 @@ engine-wire adapter storage-provider adapter +--------- optional Binding -------+ ``` -The current Redis provider adapter owns workload and Service rendering and -emits a structured RESP binding. The engine adapter declares which bindings it -accepts. Admission rejects unsupported combinations before an engine Pod is -created. Legacy `lm` and `mooncakestore` bindings remain implemented only so -old alpha objects stay reconcilable until Phase 7. +The Redis provider adapter owns workload and Service rendering and emits a +structured RESP binding. The engine adapter declares whether it accepts RESP +or host-only operation. Admission rejects unsupported combinations before an +engine Pod is created. ### Cache type validation `spec.type` is a closed CRD enum containing `LMCache` and `SGLangHiCache`. -Remote-provider technology and lifecycle ownership are not cache types: -Mooncake is selected through `remoteStorage.provider`, and externally managed -infrastructure through `remoteStorage.ownership`. The API server rejects the -old `type: Mooncake` and `type: External` spellings before admission. +Remote-provider technology and lifecycle ownership are not cache types. Redis +is selected through `remoteStorage.provider`, and externally managed +infrastructure through `remoteStorage.ownership`. Current managed and external Redis examples are available in [`config/samples/cachebackend-lmcache.yaml`](../../config/samples/cachebackend-lmcache.yaml) @@ -129,25 +135,18 @@ and [`config/samples/cachebackend-external.yaml`](../../config/samples/cacheback | `type` | enum | Engine-side cache implementation: `LMCache` or `SGLangHiCache`. Defaults to `LMCache`. | | `lmCache` | object | Typed LMCache MP configuration: topology, chunk size, and PodLocal server image/port/L1/resources. | | `remoteStorage` | object | Optional remote tier. Omitting it means host-only and provisions no provider workload. | -| `remoteStorage.provider` | enum | Current MP provider: `Redis`. `LMCacheServer` and `Mooncake` remain in the alpha schema only for legacy compatibility until Phase 7. | +| `remoteStorage.provider` | enum | Current MP provider: `Redis`. | | `remoteStorage.ownership` | enum | `Managed` or `External`. | -| `remoteStorage.endpoint` | string | Required for `External`, rejected for `Managed`; managed endpoints are controller-observed in status. Current Redis requires bare `host:port`. Legacy providers retain their old scheme validation only while the compatibility implementation exists. | +| `remoteStorage.endpoint` | string | Required for `External`, rejected for `Managed`; managed endpoints are controller-observed in `status.remoteStorage`. Redis requires bare `host:port`. | +| `remoteStorage.workload` | object | Pod scheduling and security for a `Managed` provider workload. Rejected for `External`; deliberately has no generic replicas/autoscaling fields. | | `remoteStorage.redis` | object | Redis-owned image and resource configuration. | -| `remoteStorage.lmCacheServer` | object | Legacy IP compatibility field; not a current MP backend. Removed in Phase 7. | -| `remoteStorage.mooncake` | object | Legacy IP compatibility field; not a current MP backend. Removed in Phase 7. | | `observation` | object | Observation-owned `modelID` and `firstEventTimeout`. | -| `deploymentKind` | enum | Managed workload kind: `Deployment` or `StatefulSet`. Defaults to `Deployment`. | -| `replicas` | integer | Desired managed backend replicas. Defaults to `1`. Minimum `0`. See [Defaulting](#defaulting-mutating) for the interaction with `spec.autoscaling.minReplicas` (first-apply-only). | -| `autoscaling.minReplicas` | integer | Lower bound for HPA replica count. Auto-defaulted to `spec.replicas` on FIRST APPLY ONLY by the admission defaulter when `spec.autoscaling` is set and `minReplicas` is left unset (see [Defaulting](#defaulting-mutating) for the first-apply-only semantics); subsequent edits to `spec.replicas` do NOT move this floor. Minimum `1`. | -| `autoscaling.maxReplicas` | integer | Upper bound for HPA replica count. Required when `autoscaling` is set. Minimum `1`. Cross-field validation: `minReplicas <= maxReplicas`. | -| `autoscaling.targetCPUUtilizationPercent` | integer | Target average per-pod CPU utilization for the HPA. Defaults to `80` when unset. Range `[1, 100]`. | | `integration.mode` | enum | Which cache tiers the engine is wired for: `Offload` (default) or `EventsOnly`. `Offload` is full participation — cache-aware routing (tier-1) plus the KV-offload connector (tier-2). It may remain host-only, connect to externally owned remote storage, or provision a provider workload when `remoteStorage.ownership` is `Managed`. `EventsOnly` wires routing only: the kvevent-subscriber sidecar is injected when the controller runs with `--kvevent-subscriber-image` set and `observation.modelID` is present; otherwise the append is skipped fail-open. No KV connector or backend server is created. See [Events-only mode](#events-only-mode-specintegrationmode--eventsonly). | | `integration.role` | enum | Engine participation mode: `ReadOnly`, `WriteOnly`, or `ReadWrite`. Defaults to `ReadWrite`. LMCache currently admits only `ReadWrite`; directional roles remain reserved for a connector that demonstrably enforces them. | -| `integration.failOpen` | boolean | Default `true`. When `true`, engine pods fall back to local prefill on cache unreachability — the cache is an optimization, never a serving dependency. Setting it to `false` is an advanced opt-in to fail-closed serving (the cache becomes a serving dependency); the controller surfaces this as a Warning Kubernetes Event on the owning `CacheBackend`. **Pair-specific exception — `(sglang, LMCache)`:** SGLang has no cacheless code path while `--enable-lmcache` is on, so its co-scheduled MP worker is a *serving prerequisite* (a worker that never starts wedges the engine), not a remote dependency that degrades to local prefill. `failOpen` is still honored at the tier that can actually be "unavailable" — the shared L2 (the worker comes up L1-only when Redis is unreachable). This is a documented, accepted boundary; see the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md) and [SGLang engine support](#sglang-engine-support). | +| `integration.failOpen` | boolean | Default `true`. Remote L3 failure is soft by default and the PodLocal MP server can continue L1-only. The co-scheduled `lmcache-mp-server` is part of the required connector path for both vLLM and SGLang, so `failOpen` does not turn a missing or broken PodLocal server into a cacheless engine launch. Setting `false` makes remote storage a serving dependency and is surfaced as a Warning Event. | | `integration.engineOverrides` | object | Optional engine-injection overrides applied to the args/env the pod-mutating webhook would otherwise inject into the engine container. See [Engine-injection overrides](#engine-injection-overrides-specintegrationengineoverrides). | | `engineSelector.matchLabels` | map | Equality-based label selector matched against engine **pod** labels (the pod template's `metadata.labels`, not Deployment, DaemonSet, or any other workload-level labels). Every key/value here must appear on the pod for it to match. `matchExpressions` is intentionally not exposed in v1alpha1 — the surface is `matchLabels` only. | | `hiCache` | object | Typed SGLang native HiCache configuration. Required only for `type: SGLangHiCache`; see [SGLang native HiCache](#sglang-native-hicache). | -| `template` | object | Optional pod-level overrides for managed backend pods. This is a narrow override surface, not a full `PodSpec`; backend containers come from controller defaults. | | `allowCrossNamespace` | boolean | Opt-in flag that allows `spec.remoteStorage.endpoint` to resolve to a Kubernetes Service in a different namespace from the CacheBackend itself. Without it, admission rejects cross-namespace Service-DNS endpoints. External hostnames and IPs are unaffected. Defaults to `false`. | > **Per-namespace lookup tuning lives on CachePolicy, not CacheBackend.** The @@ -156,32 +155,21 @@ and [`config/samples/cachebackend-external.yaml`](../../config/samples/cacheback > which are the surfaces actually wired into the server's `ResolvedPolicy` and > the `LookupRoute` path. -### Template Overrides - -`spec.template` supports partial pod-level overrides that can be merged with managed backend defaults: - -- `nodeSelector` -- `affinity` -- `tolerations` -- `topologySpreadConstraints` -- `imagePullSecrets` -- `serviceAccountName` -- `securityContext` -- `priorityClassName` -- `schedulerName` -- `runtimeClassName` -- `terminationGracePeriodSeconds` - -It intentionally does not expose `containers`; requiring users to provide containers would conflict with managed backend defaults and would make simple scheduling overrides unnecessarily large. - ### Resources Current resources live with the workload owner: `lmCache.podLocal.server.resources` for the MP native sidecar and -`remoteStorage.redis.resources` for managed Redis. Legacy -`remoteStorage.lmCacheServer.resources` and `remoteStorage.mooncake.resources` -remain renderer inputs only until Phase 7. Provider renderers deep-copy the -selected block onto their managed container. +`remoteStorage.redis.resources` for managed Redis. The Redis renderer +deep-copies the selected block onto its managed container. + +Managed provider Pod placement and security are configured independently under +`remoteStorage.workload` (`nodeSelector`, affinity, tolerations, topology spread, +image-pull secrets, ServiceAccount, Pod security context, priority/scheduler, +runtime class, and termination grace). Admission rejects this block for +`ownership: External`, because inference-cache does not own that workload. +There is intentionally no generic replica count: the current managed Redis is a +standalone singleton, while a real Redis Cluster needs provider-specific shard, +replica, discovery, and resharding semantics. **Pass-through to the rendered container.** The provider adapter `DeepCopy`'s the selected typed resource block onto `Container.Resources`. The deep copy is @@ -194,8 +182,6 @@ provider derives `--maxmemory` from `remoteStorage.redis.resources.limits.memory` at roughly 80%, with `allkeys-lru`. -**Autoscaling CPU-request fallback.** A `targetCPUUtilizationPercent` HPA needs a **positive** CPU request as the denominator for its utilization math, so when `spec.autoscaling` is set the adapter fills in `cpu: 250m` whenever the selected provider resource block's `requests.cpu` is absent OR non-positive. The non-positive case matters because the admission validator admits `requests.cpu: "0"` as a valid kubelet shape (an explicit "no guaranteed minimum" for non-autoscaled pods); without the autoscaling-side replacement, the HPA would dial against a 0 denominator. A positive operator-supplied value (e.g. `requests.cpu: "1"`) survives untouched. The fallback is **CPU-only** — it never synthesises a memory request — and the operator-supplied memory block (or the legacy webhook/provider default) flows through unchanged. - **`resources.claims` is rejected at admission.** `corev1.ResourceRequirements` also exposes a `Claims` slice for Dynamic Resource Allocation (DRA), but the renderer does not plumb the matching pod-level `spec.resourceClaims` — a claim-bound `container.resources.claims` would render a pod the apiserver rejects (claim name doesn't resolve at the pod level). The validating webhook (`rejectResourceClaims`) hard-rejects non-empty `claims` until DRA is wired end-to-end; a nil/empty `claims` slice admits unchanged. **Request/limit relationship is resource-aware.** The validating webhook (`rejectResourceLimitsBelowRequests`) enforces K8s' two-regime contract: @@ -418,8 +404,8 @@ The KV-event subscriber reads its model identity from **What events-only does and does not provision.** An events-only backend is the lighter, routing-only deployment: -- **No provisioned server.** The reconciler creates no Deployment and no Service for an events-only backend, and `status.endpoint` stays empty (there is no server address to publish). Flipping an existing `Offload` backend to `EventsOnly` sheds the previously-provisioned Deployment + Service on the next reconcile. -- **No KV connector.** The pod webhook does NOT inject the `--kv-transfer-config` arg or the `LMCACHE_*` env into the engine container — the engine container is left otherwise untouched. Because nothing dials a cache server, no endpoint is required, and the webhook injects an events-only engine pod even though `status.endpoint` is empty (the usual empty-endpoint fail-open is bypassed for this mode). +- **No provisioned server.** The reconciler creates no Deployment or Service and leaves `status.connector` and `status.remoteStorage` absent. +- **No KV connector.** The pod webhook does not inject runtime connector arguments or an MP server. It may append only the observation subscriber described below. - **Mode wins over host-tier configuration.** If `spec.lmCache` is present, `EventsOnly` still injects no LMCache connector or host-tier settings; the block is ignored for engine wiring. Operators should omit `spec.lmCache` on routing-only resources so the manifest does not imply an active host tier. `spec.remoteStorage` is rejected rather than ignored because it declares a provider that nothing would dial. - **The kvevent-subscriber sidecar is injected — when wired.** That is the whole point of routing: once the sidecar is appended, `LookupRoute` and the per-backend `status.indexParticipation` slice behave identically to a managed backend; only the offload tier (server + connector) is absent. The append is gated exactly as for a managed backend and is skipped **fail-open** when either gate is unmet: the controller must run with `--kvevent-subscriber-image` set (unset by default, so a default install injects no subscriber) AND `spec.observation.modelID` must be present to supply `--model-id`. When skipped, the webhook leaves the engine pod untouched and stamps no `injected-by` annotation. - **Evictions are tier-aware.** The subscriber tags each prefix with a cache tier from the block lifecycle: `BlockStored` → **T1** (resident in HBM). On a `BlockRemoved`, the two modes diverge. In `Offload` mode the paired LMCache L2 tier still holds the block after the engine evicts it from HBM, so the subscriber (`--ignore-block-removed=true`) **re-reports the evicted prefix at tier T2** (reload-able from host RAM), anchored at the eviction timestamp — the entry is *kept*, not dropped, and honestly tagged colder than HBM; a later `BlockStored` of the same content re-reports it back at T1. In `EventsOnly` mode there is no L2 retaining the block, so a `BlockRemoved` genuinely means the prefix is gone and the hint MUST be pruned — the subscriber omits the flag and forwards the eviction as `PREFIX_EVICTED`. Either way a stale/mis-tagged hint is soft state (a cache miss at worst, never a wrong answer). See `docs/design/kvevent-subscriber-wiring.md` "L2 cache tier semantics". @@ -431,41 +417,20 @@ The KV-event subscriber reads its model identity from **Admission constraints.** Because an events-only backend provisions no server, server-shaped configuration is structurally meaningless and is rejected at admission: - `spec.remoteStorage` is forbidden — any Managed or External declaration requests an offload provider that events-only deliberately does not wire. -- `spec.autoscaling` is forbidden — there is no workload to scale. The rejection is field-scoped to `spec.autoscaling`. - -### LMCache server / client version alignment -The standalone lmcache-server image and the **lmcache client** compiled into the -engine image (operator-supplied, or pip-installed into the engine at runtime) -communicate over a versioned wire protocol. **They must be wire-compatible.** A -mismatch does not fail loudly: remote KV stores fail (e.g. `[Errno 32] Broken -pipe` / connection resets), the backend records 0 reload hits, and tier-2 -(remote KV offload) is **silently disabled** with no surfaced error. The cache -plane keeps serving and routing; it simply never gets a tier-2 hit, which is -hard to distinguish from a cold cache. +### LMCache MP server / client version alignment -The controller resolves the managed server image in this order: +The digest-pinned `spec.lmCache.podLocal.server.image` and the LMCache client +inside the operator-owned engine image must expose compatible MP APIs. The +controller does not own or rewrite the engine image, and it does not use an +image allowlist or verifier init container. Normal engine startup is the +authoritative connector/package compatibility check. -1. `spec.remoteStorage.lmCacheServer.image`, for a per-CacheBackend override. -2. The controller's `--lmcache-server-image` flag, for an operator-selected - deployment default. - -There is no image version compiled into the Go binary. When both settings are -empty, managed LMCache rendering fails with a configuration error instead of -creating a Pod with an empty image. The shipped Kustomize install sets -`--lmcache-server-image=lmcache/standalone:v0.4.7` in -`config/manager/manager.yaml` as its reproducible baseline. Operators should -override that deployment argument (or the equivalent value in their Helm -packaging) to match the client in their engine image; a CR-level image remains -authoritative when one backend needs a different version. - -The **same silent store-failure signature can also come from an under-provisioned server that is OOMKilled under load** — the standalone server keeps KV in memory, and a default memory request far below a large model's working-set KV (e.g. a 32B model's KV is tens of GB) will OOM the server the moment stores begin, dropping every connection. Size the server's memory to the expected working set. (Surfacing tier-2 store-failure / hit-rate health so neither failure mode stays silent is a separate follow-up.) - -Because of this: - -- The shipped deployment baseline is **pinned to a specific, non-floating version**, never `:latest`. A floating tag can drift to a server build whose wire protocol no longer matches the client, reintroducing the silent-disable failure mode on an unrelated pull. The baseline tag `v0.4.7` is version-aligned with the validated lmcache 0.4.7 client, but the standalone server image was not independently wire-tested; confirm against a tested build — ideally an `@sha256:` digest — before release. -- **Pin both sides.** When an operator sets `--lmcache-server-image` or overrides `remoteStorage.lmCacheServer.image`, they must choose an lmcache-server version that is wire-compatible with the lmcache client version their engine image carries, and pin the engine's client too (a `pip install lmcache` at engine startup is itself a floating reference). For non-local runs, prefer an `@sha256:` digest. -- IC **cannot auto-match** these versions: it has no source of truth for the engine's client version (the engine image is operator-supplied and the client may be pip-installed at runtime), so it cannot detect or warn on a skew today. The mitigation is this alignment contract plus an operator-selected pinned deployment baseline; runtime detection / a tier-2 health signal is a separate follow-up. +Pin the MP server image by digest and pin the engine image/package set through +the inference system's own release process. A connector mismatch is surfaced +through the engine Pod's startup failure and the advisory +`EngineCompatibility` observation; inference-cache cannot prove package +compatibility from image names alone. ### LMCache client kernels ↔ engine-image CUDA / vLLM alignment @@ -555,121 +520,54 @@ Extending the check to SGLang is a follow-up. server-side cache path): the kernel check catches the engine-side load cause that the round-trip probe cannot see. -### Legacy IP Mooncake provider configuration (compatibility only) - -> This section documents implementation retained until Phase 7 so old alpha -> objects remain diagnosable. Mooncake-through-LMCache/IP is not a current -> production path, is not a sample, and must not be translated to Redis -> automatically. Operators must explicitly choose typed host-only MP or Redis. - -`spec.remoteStorage.mooncake` selects the Mooncake provider adapter -(`internal/adapters/builtin/storage/mooncake.go`) to reconcile the standalone -**Mooncake master** workload. The vLLM runtime adapter -separately wires engine pods to it through the LMCache remote-binding contract. Mooncake is -historically represented a durable/shared cache path in -[`docs/design/lmcache-server-persistence.md`](lmcache-server-persistence.md) -(the old in-memory IP server and Mooncake mapping are both legacy; this text is -preserved only to explain the compatibility renderer). - -> **Operator requirement — the Mooncake master runs on the host network.** Unlike LMCache's `lm://` (one server, one port, one connection — a virtual ClusterIP suffices), Mooncake is a **peer-to-peer transfer-engine mesh**: the master on `:50051` returns only a directory pointer ("this block lives on node B"), and the engine then dials that node's real IP on a **dynamically negotiated port** to move the KV bytes. A ClusterIP Service forwards only the ports declared on it, and CNI overlay pod IPs are not reachable for the mesh — so the adapter renders the master with `hostNetwork: true` behind a **headless** Service (`clusterIP: None`), whose DNS name (published as `status.endpoint`) therefore resolves straight to the master's node IP with every port reachable. Consequences you must plan for: -> -> * The namespace must **permit `hostNetwork`** — a Pod Security `restricted` namespace will reject the master pod. -> * The master **reserves its ports (50051 / 8080 / 9003) on its node** (the API server defaults `hostPort=containerPort` for hostNetwork pods), and its Deployment uses the `Recreate` rollout strategy — a rolling surge would collide on those ports. -> * The master is a **singleton**. `spec.replicas > 1` and `spec.autoscaling` are **rejected at admission** when `remoteStorage.provider: Mooncake`: a second replica either fails to schedule because its node ports are already bound or comes up as an independent master and silently splits the store. `spec.replicas: 0` (disabled) and `1` remain valid. -> * **Network exposure — plan for it.** Host networking publishes the master's RPC (`50051`), metadata (`8080`) and metrics (`9003`) ports, plus the transfer engine's dynamically negotiated data ports, directly on the **node's interfaces**, outside the pod network. `NetworkPolicy` selects pods by pod IP and therefore **does not constrain a hostNetwork pod's listeners** — the isolation you get from pod-network policy is simply absent here. Restrict access with node-level controls instead: security-group / firewall rules on the node interfaces, and by constraining which nodes the master and its engines may schedule onto. Treat all of these ports as cluster-internal only; none of them authenticate callers. -> * **Engine pods need host networking too — opt in with `spec.integration.engineHostNetwork: true`.** Mooncake's mesh is dialed *from* the engine, so an overlay engine pod cannot participate. With the flag set, the Pod webhook moves matched engine pods onto the host network (`hostNetwork` + `dnsPolicy: ClusterFirstWithHostNet`) alongside the usual `LMCACHE_*` wiring. Until it is set, admission **warns on every Mooncake `remoteStorage` apply** and the backend reports `Ready` while transferring **zero KV**. -> -> It is opt-in, never injected by default, because it rewrites the networking of a pod **you** own. `hostNetwork` is a privilege, and mutating webhooks run **before** Pod Security validation — so silently adding it would turn a working engine pod into one a `restricted` namespace *rejects*, with an error naming Pod Security rather than this controller. The flag is rejected on backend types that do not need it, so it can never sit inert. -> -> **Setting the flag does not move pods that already exist.** Injection happens at pod *admission*, and a Pod's `hostNetwork` is immutable — so enabling it changes only pods admitted afterwards. **Roll your engine workload** (`kubectl rollout restart deployment/`) after enabling it, or the running engines stay on the overlay and keep transferring zero KV. -> -> Host networking is applied **together with** the `LMCACHE_*` connector, behind the same gate: if the backend has not yet published `status.endpoint`, a matched engine pod admits *un-wired* — no connector **and** no `hostNetwork`. It is never granted to a pod that has nothing to use it for. Such a pod needs a roll once the backend reports `Ready` in any case, since it is missing the connector env too. -> -> * **Engine scheduling and rollout, under host networking.** These constraints land on **your** engine Deployment, which this controller does not own and therefore cannot clamp the way it clamps the master: -> * The API server defaults `hostPort` to `containerPort` for hostNetwork pods, so **each engine replica reserves its serving port (e.g. `8000`) on its node** — at most one engine replica per node per port. Size the engine's replica count against schedulable nodes, not just GPUs. -> * A `RollingUpdate` engine Deployment can **deadlock**: the surge pod cannot bind a port the outgoing pod still holds, so it stays `Pending` forever and the rollout never completes. Use `strategy: Recreate` (or `maxSurge: 0`) on a hostNetwork engine Deployment. -> * Pod-network isolation is absent for engine pods too — the same `NetworkPolicy` caveat above applies to them. -> -> This is inherent to Mooncake, not a choice the adapter can avoid. Host-only LMCache and the standalone LMCacheServer provider are unaffected and stay on the pod network. - -**Mooncake is wired as an LMCache *remote backend*, not vLLM's native MooncakeStoreConnector.** The engine runs the *same* LMCache connector the LMCache backend uses (`kv_connector=LMCacheConnectorV1`) pointed at a `mooncakestore://host:port` remote store — the Mooncake analog of `lm://`. So the engine-side injected wire follows the same [pod-webhook engine-wiring contract](#mutating-pod-webhook-engine-wiring) **except** that `LMCACHE_REMOTE_URL` carries the `mooncakestore://` scheme. The native `MooncakeStoreConnector` is configured exclusively through a `MOONCAKE_CONFIG_PATH` JSON file (it has no env-var surface for the master address), and the pod-mutating webhook can only inject env + args — it cannot write a file into a user-owned engine container — so routing the controller-resolved master endpoint through `LMCACHE_REMOTE_URL=mooncakestore://…` is the only path that lets `status.endpoint` reach the engine via injection alone. Operators who prefer the native connector pre-bake their own config file; this adapter targets the auto-wired path. - -Provider-side fields consumed by `provider.ResolveMooncakeServer`: - -| Field | Default | Purpose | -|---|---|---| -| `remoteStorage.mooncake.image` | `docker.io/kvcacheai/mooncake:0.3.11.post1` *(pinned, non-floating; fully qualified)* | Container image for the standalone Mooncake master. Fully qualified (`docker.io/…`) so CRI-O nodes without short-name resolution configured do not reject it; it is version-aligned with the `mooncake-transfer-engine` 0.3.11.post1 release on PyPI. Pin to an `@sha256:` digest for non-local runs. | -| `remoteStorage.mooncake.command` | `mooncake_master --rpc_port=50051 --metrics_port=9003 --enable_http_metadata_server=true --http_metadata_server_host=0.0.0.0 --http_metadata_server_port=8080` | Master command and arguments. The default launches RPC, Prometheus metrics, and the embedded HTTP metadata server. **Do not change the RPC (50051) or HTTP metadata (8080) ports through this override**: the rendered Service, readiness probe, status endpoint, and engine binding use those fixed values and are not derived from free-form command text. | -| `remoteStorage.mooncake.resources` | memory request `4Gi`, limit `8Gi` | Resources for the managed master container. An explicit typed block replaces the defaults; autoscaling also supplies a `250m` CPU request when no positive CPU request is present. | - -The Service exposes the master's **RPC port (50051) first** so the reconciler's engine-agnostic `serviceEndpoint` helper publishes it into `status.endpoint`, plus the HTTP metadata port (8080). +### Removed IP and Mooncake history -Engine-side: the adapter injects the same `--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":""}'` arg and the same `LMCACHE_*` / `VLLM_USE_V1` / `INFERENCECACHE_FAIL_OPEN` / `PYTHONHASHSEED` env as an LMCacheServer binding, with `LMCACHE_REMOTE_URL=mooncakestore://`. Chunk size, serializer, and host-memory settings come from `spec.lmCache`. The reserved args/env are therefore identical. The kvevent-subscriber sidecar is also identical (the KV-event stream comes from vLLM, not the L2 store; `--hash-scheme=vllm`, `--ignore-block-removed=true`). - -**Transfer-engine tuning is operator-supplied, not env-injected.** Mooncake's static transfer-engine config (`metadata_server`, `protocol` tcp/rdma, `device_name`, segment sizes) lives in LMCache's `extra_config`, which is read from an engine-side config file (`LMCACHE_CONFIG_FILE` / `MOONCAKE_CONFIG_PATH`) — not from env vars, so the webhook cannot inject it. The adapter wires the controller-resolved master address + the connector; the transfer-engine defaults (P2P-handshake metadata) cover the simplest deployment, and operators provide a config file for a real RDMA / HTTP-metadata setup. A kind reference stack that validates the end-to-end Mooncake deployment shape (the A2-equivalent of the LMCache reference stack) is a tracked follow-up. The master image entrypoint + RPC/metadata/metrics ports are now confirmed on a live cluster; until that stack lands, treat the `extra_config` transfer-engine defaults and the full end-to-end deployment shape as not-yet-cluster-validated. +The former standalone LMCache IP server and Mooncake-through-LMCache provider +were physically removed in Phase 7. They are not accepted by the served CRD and +must not be translated to Redis automatically because that would change +sharing and durability semantics. Historical rationale remains in +[the migration roadmap](lmcache-multiprocess-migration-roadmap.md) and +[lmcache-server-persistence.md](lmcache-server-persistence.md). ## Status -| Field | Type | Purpose | -|---|---|---| -| `endpoint` | string | Observed endpoint clients should use. For External ownership this mirrors `spec.remoteStorage.endpoint`; for Managed remote storage it is populated from the controller-rendered Service. It stays **empty** for host-only and events-only backends because neither has a remote provider address to publish. | -| `matchedEnginePods` | integer | Snapshot count, at the last reconcile, of pods in the CacheBackend's namespace whose labels satisfy `spec.engineSelector`. Pointer in Go so nil ("not yet computed") is distinguishable from an observed `0` ("computed and zero pods match"). Refreshed at reconcile cadence — not a real-time per-pod counter. The steady cadence is 30s; during known churn the reconciler uses a conditional 5s cadence when the observed matching Pod count differs from the desired replica sum of Deployments whose pod-template labels match the selector. This keeps the no-Pod-watch design while reducing stale operator output during rolling restarts. The field stays nil when no claim-capable selector is configured — both when `spec.engineSelector` is absent AND when `spec.engineSelector.matchLabels` is present but empty (the webhook treats an empty match map as no-claim by design, so the count is no-claim too). A CR that previously had a non-empty selector and just lost it gets its prior value cleared back to nil so the printer column does not advertise a stale match. | -| `engineSelectorMessage` | string | Operator-facing diagnosis for selector drift. Set when `spec.engineSelector.matchLabels` is configured and `matchedEnginePods` is observed as `0` while engine pods are expected; the message echoes the selector (`spec.engineSelector.matchLabels={...}`) and states that no Pods in the namespace match. If the selector matches a Deployment that is intentionally scaled to zero, `matchedEnginePods` still reports the observed `0`, but this message stays empty because no engine pods are expected. Cleared once at least one pod matches, the matching Deployment is scaled to zero, or the selector is removed. The controller also emits a Normal `EngineSelectorUnmatched` Event when the initial observation is zero, when a previously non-zero match count transitions to zero, or when upgrading an existing zero-count status that did not yet have the diagnostic message; steady-state zero with an unchanged message does not re-emit. | -| `failOpen` | boolean | Observed echo of the effective `spec.integration.failOpen`. Represented as a pointer in Go so an explicit `false` is serialized and operators can read the current mode from status alone. | -| `indexParticipation` | object | Per-backend slice of the cluster-wide cache index, projected from the server's `/snapshot` by grouping replicas by owning `CacheBackend`. Populated by the CacheIndex poller (status-only). Object is unset until the poller has observed a successful scrape that names the backend's replicas (see [Index Participation](#index-participation)). | -| `firstKVEventObservedAt` | time | Write-once latch: the first time the [KV-event readiness gate](#kv-event-readiness-gate) observed `indexParticipation.lastEventAt` populated. This is the durable "have we EVER seen a KV event" signal — `lastEventAt` itself is a current-view projection the poller legitimately clears when a backend's replicas drain, so reading it alone would let a backend that already passed the gate regress. Set write-once by the controller and never cleared (a monotonic marker). It is inert while the backend is not managed (External / unsupported runtime) and is intentionally left in place there — clearing it would be ineffective anyway, since the preserved poller-owned `lastEventAt` would immediately re-satisfy the gate on a return to the managed path — so a return to managed stays Ready without re-gating, consistent with the "ever observed" contract. | -| `firstAvailableAt` | time | Write-once latch: the **stable anchor** for the `firstEventTimeout` clock. For a managed (`Offload`) backend it latches the first time the managed cache-backend workload was observed `Available` — used instead of the live Deployment `Available` condition's `LastTransitionTime` precisely because that resets on an availability flap. For an `EventsOnly` backend there is no workload to wait on, so it latches at the first reconcile (the clock starts immediately). Anchoring on this monotonic value keeps the elapsed window growing WITHIN a serving mode, so once a backend breaches the timeout (`Degraded`/`NoKVEventsObserved`) a later flap cannot bounce it back to `AwaitingFirstKVEvent` — it stays Degraded until an event arrives. It is stable across flaps and a recreated managed Deployment, but NOT across a mode change: a server-bearing→`EventsOnly` flip re-anchors it to the flip moment (and bypasses the sticky `NoKVEventsObserved` reason) so the flip gets a fresh first-event window rather than inheriting the old mode's availability time or timed-out verdict, and an unmanaged transition clears it so a later re-entry starts fresh. | -| `observedServerInstance` | string | The controller's **cascade-decision baseline** — a stable identifier for the Ready cache-server pod set the controller last anchored against. NOT a live "current pod set" view: it is intentionally pinned through transient rolling-update midpoints and through no-Ready windows so the cascade does not fire on rollbacks or transient outages. For the current matched-pod inventory operators should consult `status.matchedEnginePods` (engine side) and `kubectl get pod` (cache-server side). Shape: `:` per Ready pod, comma-joined and lex-sorted by pod name; `` sums `pod.status.containerStatuses[].RestartCount` filtered to the cache-server's own containers (the container names declared on the owned Deployment's pod template). Sidecars injected by other admission webhooks (service-mesh proxies, Datadog, etc.) appear in `containerStatuses` but are absent from the template and are intentionally excluded — a sidecar crash-loop must not advance the identifier and roll the engine fleet. An in-place restart of a cache-server container (kubelet respawning a crashed container — OOM with `restartPolicy=Always` reuses pod.UID) DOES advance the identifier and is observable. On a transition that reflects an actual replacement (a prior pod is gone, or a persisting pod's restart-sum advanced — NOT a rolling-update strict-superset midpoint) the reconciler cascade-restarts every engine Deployment that owns pods carrying this backend's `inferencecache.io/injected-by` + matching `injected-by-uid`, by patching `inferencecache.io/cache-server-restart-trigger` onto each Deployment's pod template — the same mechanism `kubectl rollout restart` uses. Rate-limited to once per ~30s per backend. Empty until the first Ready pod; empty→set never cascades (there is no prior server-instance to invalidate, so any engines that connected during the empty window are connecting to the very pod now being baselined). **Strict-superset transitions are persisted as the new baseline ONLY when the owning Deployment is converged** (`spec.replicas == status.readyReplicas == status.updatedReplicas == len(live Ready pods)` AND `observedGeneration >= metadata.generation` — the live-count clause cross-checks the Deployment's reported state against the pod list this reconcile actually saw, so a stale `status.readyReplicas=1` while two pods are mid-rollout cannot fake convergence) — a converged steady-state widening is an operator-driven scale-up, so the added pods must enter the baseline or a later replacement of just an added pod would still look like a strict superset and miss the cascade. Strict-superset transitions where the Deployment is NOT converged are rolling-update midpoints; persisting them would let a rolled-back rollout (new pod briefly Ready, then killed, leaving the original pod alone) look like "the new pod was replaced" and false-cascade. **Stale-while-unavailable**: when no Ready cache-server pod exists at all (Deployment scaled to 0, mid-rollout, image-pull stuck), this field intentionally retains its prior value rather than clearing — clearing would turn the eventual recovery's `""` → `new-uid:0` transition back into a first-observation baseline and silently skip the cascade, defeating the controller's purpose. Inert and cleared on every transition out of the managed-provider path: External ownership, host-only caching, events-only mode, and unsupported runtimes. The in-process cascade shadow is wiped alongside the field on each of these paths so a later return to a managed provider starts from a clean baseline rather than the prior-period UID. Operator-side recovery for the upstream LMCache `LMServerConnector` EPIPE-on-restart bug — see [LMCache/LMCache#3565](https://github.com/LMCache/LMCache/issues/3565). | -| `observedGeneration` | integer | The `.metadata.generation` last reconciled by the controller. Lets clients tell whether the observed status reflects the current spec. | -| `conditions` | array | Kubernetes conditions keyed by `type`. See [Conditions](#conditions). | - -### Conditions - -The set of published condition types depends on the backend's integration mode and type: +Status separates the Pod-local connector from the optional network-addressable +remote tier: -- **Offload-managed backends** (`spec.integration.mode=Offload` on a managed type, where the controller renders a Deployment + Service) publish up to seven: `Ready`, `Degraded`, `Progressing`, `FunctionalProbeOK`, `EngineKernelsHealthy` (when a matched engine pod runs the lmcache kernel-check), `T2Degraded` (once a tier-2/LMCache backend has been exercised), and `EngineCompatibility` (when an injected engine pod is observed crash-looping after connector injection). -- **Host-only backends** (resources with no `spec.remoteStorage`) publish `Ready`, `Degraded`, and `Progressing`, plus the engine-side advisory conditions when applicable. Their endpoint stays empty. `HostOnlyActive` is the base `Ready=True` reason before the KV-event gate overlays `AwaitingFirstKVEvent`, `KVEventsObserved`, or `NoKVEventsObserved`. -- **Events-only backends** (`spec.integration.mode=EventsOnly`) publish exactly three: `Ready`, `Degraded`, `Progressing`. `FunctionalProbeOK`, `T2Degraded`, `EngineKernelsHealthy`, and `EngineCompatibility` are **Offload-managed-only** and are **never** published on an events-only backend — there is no provisioned server to functionally probe, no tier-2 offload to mark degraded, no LMCache native kernels to check, and no injected KV connector that could be incompatible (events-only injects none); an Offload→EventsOnly flip clears all four (see [Events-only mode](#events-only-mode-specintegrationmode--eventsonly)). -- **Externally owned remote storage** publishes `Ready` + `Progressing` only (there is no rollout to degrade and no probe to drive; the operator manages the provider out-of-band and the controller only validates and mirrors the endpoint). - -The `Ready` / `Degraded` / `Progressing` semantics below apply to both Offload-managed and events-only backends (an events-only backend has no workload to roll out, so it is "up" the moment it exists and the KV-event gate starts immediately — see [Events-only mode](#events-only-mode-specintegrationmode--eventsonly)); the `FunctionalProbeOK`, `T2Degraded`, `EngineKernelsHealthy`, and `EngineCompatibility` rows are Offload-managed-only. - -**Managed backends** (Offload-managed; the `FunctionalProbeOK` / `T2Degraded` / `EngineKernelsHealthy` / `EngineCompatibility` rows do not apply to events-only): - -| Type | Meaning | +| Field | Purpose | |---|---| -| `Ready` | True once the backend Deployment has rolled out its current generation, has enough updated + available replicas to serve traffic, **and** — when the [KV-event readiness gate](#kv-event-readiness-gate) applies — at least one KV event has been observed for the backend (reason `KVEventsObserved`), **and** — when the [functional-probe gate](#functional-probe-gate) applies — the most recent probe call succeeded across every stage the backend runs. Workload Available but no event yet is `Ready=False`, reason `AwaitingFirstKVEvent`. Workload Available and KV-event observed but the probe reported a stage failure is `Ready=False` with reason `ProbeIngestFailed` / `ProbeRoutingFailed` / `ProbeT2Failed`. Deployment-level reasons: `BackendReady` (both gates disabled and Available), `RolloutInProgress`, `ScaledToZero`, `ReplicasUnavailable`. **And** — when a matched engine pod admitted in **strict** kernel-check mode reports a kernel load failure — `Ready=False` with reason `EngineKernelDegraded` (see [`EngineKernelsHealthy`](#conditions)); report-only mode never downgrades `Ready`. The `BackendDegraded` / `BackendRecovered` Events narrate the `ReplicasUnavailable` → `BackendReady` / `KVEventsObserved` transitions. | -| `Degraded` | True when the backend is in a terminal unhealthy state: rolled out but replicas unavailable (reason `ReplicasUnavailable`), or the managed workload is Available but no KV event observed within `firstEventTimeout` (reason `NoKVEventsObserved`). False (`NotDegraded`) otherwise. The functional-probe gate does NOT participate in `Degraded` — a probe failure is reflected only in `Ready` and `FunctionalProbeOK`, leaving `Degraded` reserved for managed-Deployment health (so an operator can tell "the probe says the cache plane is broken" apart from "the workload itself is in a terminal state"). | -| `Progressing` | True while the controller is still driving the live state toward the desired state (rollout in flight, first apply, awaiting first KV event). False once converged (`Synced`), stuck (`Degraded`), or scaled to zero (`ScaledToZero`). The pair (`Ready=False`, `Progressing=True`) means "still converging"; (`Ready=False`, `Progressing=False`) means "stuck/degraded" (or scaled to zero). | -| `T2Degraded` | **Advisory** tier-2 (external offload, e.g. LMCache) health, derived from `status.indexParticipation.t2HitRate` (written by the CacheIndex poller). Published only once the tier has been **exercised** (external lookups observed): `True`/`T2ZeroHitRate` when it was queried but served **zero** reloads (wired but useless — a store/connection failure, an under-sized remote server, or a scheduler/worker hash mismatch); `False`/`T2Serving` when it is serving reloads (hit-rate > 0). Absent entirely until the tier is exercised (distinct from `False`). It **never gates `Ready`** — tier-2 is an optimization, not a serving dependency (fail-open). For Prometheus alerting use the `inferencecache_backend_t2_hit_rate{backend}` gauge (CR `.status` is not scraped). The signal is **lifetime-cumulative** — it flags a tier that has *never* served a reload (the silent-from-start failures: scheduler/worker hash mismatch, server OOM, version skew); a mid-life regression (served reloads before, now zero) keeps hit-rate > 0 and so does **not** trip `T2Degraded`. That windowed case is caught instead by the per-pod `LMCacheT2NoHits` alert. | -| `FunctionalProbeOK` | The most recent functional-probe outcome. `True/ProbeOK` when every enabled stage (ingest, routing, and — for LMCache — tier-2 put/get) round-tripped; `True/ProbeBypassed` when the operator opted this CR out via the `inferencecache.io/skip-functional-probe: "true"` annotation; `False/ProbeIngestFailed`, `False/ProbeRoutingFailed`, or `False/ProbeT2Failed` when the named stage failed, with the server's diagnostic in `.message`; `Unknown/ProbeError` when the controller could not reach the server's `/probe` endpoint at all (transport error, 5xx) AND no prior stage failure existed. **Sticky-False**: an HTTP error while a `False/Probe*Failed` is already published preserves the prior failure and keeps `Ready` downgraded, so a transient server outage cannot fade a known per-stage failure back to `Unknown` and then to `Ready=True`. See [functional-probe gate](#functional-probe-gate). | -| `EngineKernelsHealthy` | Engine-side native CUDA-kernel (lmcache `c_ops`) load health, read from the `lmcache-kernel-check` init container on matched engine pods. `True/KernelsHealthy` when the native kernels loaded on every reporting pod; `False/KernelLoadFailed` when one or more failed to load — a `libcudart`/CUDA-runtime mismatch (the root cause), a CPU/pure-python build with no compiled extension, or lmcache not importable; the specific cause is in the condition `.message`. In `strict` mode a `False` also downgrades `Ready` (reason `EngineKernelDegraded`); `Unknown/KernelCheckError` when a check terminated without a recognized result; `Unknown/KernelCheckPending` while a check is still running. Absent when no matched engine pod runs the check (CPU backends, annotation `off`). Default mode is fail-open observability — it does NOT gate `Ready` unless `strict`. See [client kernels ↔ image CUDA alignment](#lmcache-client-kernels--engine-image-cuda--vllm-alignment). | -| `EngineCompatibility` | **Advisory** engine↔connector observation, derived from the live container state of the engine pods this backend injected cache config into. Published `False`/`InjectedEngineCrashLooping` only when an injected engine container is in `CrashLoopBackOff` after the cache plane wired a KV connector — the live **observation**, not a confirmed root cause. A structural connector incompatibility is a common cause, canonically a **hybrid-attention model** (Qwen3.6/Next gated-DeltaNet, Mamba/Jamba, Falcon-H, Granite-hybrid, …): vLLM disables its hybrid KV-cache manager the moment any KV connector (LMCache, Mooncake, NIXL) is wired, then fails KV-spec unification at init. But a crash-loop is generic — it can equally be a bad image, command, missing dependency/secret, or OOM — so verify the cause via the engine logs. Absent when no injected engine is stuck. It **never gates `Ready`** (the engine is operator-owned; `Ready` is driven by the managed Deployment + the KV-event gate) — it names an otherwise-silent crash-loop that often sits behind a `NoKVEventsObserved` Degraded and points at the likely fix. If it is the connector (e.g. a hybrid model), the routing-preserving fix is the connector-less **events-only** integration — set `spec.integration.mode: EventsOnly` (kv-events, no offload), the supported remedy for hybrid-attention models. (Do not reach for `inferencecache.io/skip-inject`: it opts the pod out of cache wiring entirely, the kvevent-subscriber included, so it stops routing rather than preserving it.) An `InjectedEngineCrashLooping` Warning Event narrates the transition. See [supported-model matrix](#supported-model-matrix). | - -When the desired replica count is owned by an HPA (`spec.autoscaling` set) the controller compares the Ready condition against the HPA-written Deployment `spec.replicas` rather than the user-set `spec.replicas`. - -**Externally owned remote storage**: - -Resources express this shape with -`spec.remoteStorage.ownership: External`, the selected provider, and -`spec.remoteStorage.endpoint`. There is no Deployment to roll out, so provider-specific endpoint validation -is the only readiness signal the controller has. The controller mirrors the -trimmed endpoint to `status.endpoint` and publishes both conditions immediately -on every reconcile (the KV-event gate never applies to external ownership): - -| Type | Status | Reason | Meaning | -|---|---|---|---| -| `Ready` | `True` | `ExternalEndpointAccepted` | The active endpoint field is non-empty and valid for the selected provider. LMCacheServer accepts `host:port` or `lm://host:port`; Mooncake accepts `host:port` or `mooncakestore://host:port`; Redis accepts bare `host:port`. A numeric port in `1-65535` is always required, embedded whitespace and URL path/query/fragment components are rejected, and IPv6 must be bracketed. The controller provisions no provider pod for External ownership, so admission acceptance is the readiness signal. | -| `Ready` | `False` | `ExternalEndpointMissing` | The active endpoint field is empty or whitespace-only. Current admission rejects this, so the state is reachable only for a CR already stored before the webhook was installed. Status reflects the gap loudly rather than dropping the condition. | -| `Ready` | `False` | `ExternalEndpointInvalid` | The active endpoint is non-empty but fails the selected provider's shape check. Current admission rejects these values; the reason is reachable only for a CR stored before the relevant rule shipped. The message names `spec.remoteStorage.endpoint` and carries the shape error. The pod webhook applies the same validation and admits the engine pod unwired on failure. | -| `Progressing` | `False` | mirrors Ready's reason | External ownership completes admission immediately — there is no rollout the controller is still driving. Always `False`; the reason matches Ready (`ExternalEndpointAccepted` / `ExternalEndpointMissing` / `ExternalEndpointInvalid`) so `kubectl describe` shows a coherent pair. | +| `connector` | Effective MP mode/topology and matched, ready, covered, and uncovered engine/server counts. Pod-local loopback addresses are deliberately not published. | +| `remoteStorage` | Optional Redis provider, endpoint, and readiness. Absent for host-only and EventsOnly backends. | +| `matchedEnginePods` | Snapshot count for `spec.engineSelector`; nil means not yet computed, while zero is an observed no-match state. | +| `engineSelectorMessage` | Operator-facing selector mismatch diagnosis. | +| `failOpen` | Observed effective integration fail-open value. | +| `observedGeneration` | Latest CacheBackend generation reconciled. | +| `firstKVEventObservedAt`, `firstAvailableAt` | Monotonic anchors for the first-event readiness gate. | +| `indexParticipation` | Prefix count, last event, hit rate, and optional T2 hit rate projected by the CacheIndex poller. | +| `conditions` | Kubernetes conditions described below. | + +The `kubectl get cachebackend` table displays Type, Ready, Matched, the remote +Redis endpoint, Prefixes, LastEvent, and Age. It never presents the Pod-local MP +loopback address as a cluster endpoint. -Reachability of an externally owned endpoint is **not** probed by the controller; -trusting the operator is part of External ownership. A future enhancement could -degrade `Ready` on a probe failure, but that is deliberately out of scope today -(fail-soft, never a serving dependency). +### Conditions -`kubectl get cachebackend` displays a `Ready` column sourced from `status.conditions[?(@.type=="Ready")].status` (the standard K8s pattern — operators read readiness through conditions, not through a custom enum field), the observed `status.endpoint`, a `Matched` column sourced from `status.matchedEnginePods`, plus `status.indexParticipation.prefixCount` (as `PREFIXES`) and `status.indexParticipation.lastEventAt` (as `LASTEVENT`). Managed provider backends therefore show readiness, the endpoint, the operator-actionable engine-fleet count, and live index participation once reconciliation has populated them and the poller has observed a `/snapshot` tick. An empty `Matched` cell means the count has not yet been computed (e.g. cold start before the first reconcile) or the CR has no `spec.engineSelector` configured. Externally owned bindings display the operator-supplied endpoint immediately. Their `indexParticipation` is typically unset — the operator-managed provider itself has no observation sidecar — but it is not special-cased by ownership: the poller attributes replicas by engine-pod selector/annotation. An externally owned binding whose engine pods run the subscriber therefore projects `indexParticipation` the same way as a managed binding. The readiness gate still never applies to External ownership (readiness comes from endpoint acceptance), so a populated `lastEventAt` affects the displayed columns but not readiness. +- `ConnectorReady` reports whether every selected engine Pod carries the + webhook-authenticated injection record for the current generation and has a + Ready `lmcache-mp-server` native sidecar. +- `RemoteStorageReady` reports the optional Redis tier independently. Managed + Redis readiness comes from its Deployment and Service; External Redis is + accepted from the validated operator endpoint. +- `Ready` always requires the connector. Redis also gates it when + `integration.failOpen: false`; with the default fail-open behavior a degraded + L3 does not hide a healthy Pod-local connector. +- `Progressing` and `Degraded` retain the standard convergence/stuck split. + The first-KV-event, functional-probe, kernel-health, T2, and engine + compatibility conditions remain advisory or gating according to their + dedicated settings. +- EventsOnly publishes routing readiness without connector or remote-storage + status. SGLangHiCache remains engine-local and provisions no backend + workload. ### Supported-model matrix @@ -733,10 +631,18 @@ Only ONE CacheBackend ever claims a given replica — overlapping selectors must ## Contract Notes -- Lookup paths fail open by default. `spec.integration.failOpen` defaults to `true` and the engine adapter MUST fall back to local prefill on unreachability of a **remote/shared** cache tier — that tier is an optimization, never a serving dependency. **One pair-specific exception applies** to the *co-scheduled* component of the SGLang MP wire: `(sglang, LMCache)` has no cacheless engine path while `--enable-lmcache` is on, so its in-pod MP worker is a serving prerequisite (fail-open is still honored at the tier that can be unavailable — the shared Redis L2, which degrades to L1-only). See the `integration.failOpen` row above and the fail-open semantics in [`sglang-lmcache-mp-mode.md`](sglang-lmcache-mp-mode.md). Operators may opt into fail-closed serving by setting `failOpen: false`, which is loud and visible: the controller emits a Warning `FailClosedEnabled` Event on the `CacheBackend` to make it explicit that the cache has been promoted to a serving dependency. +- Lookup paths fail open by default. The co-scheduled MP server is required by + both typed LMCache engine wires and always gates connector readiness. The + optional Redis tier is the fail-open boundary: with the default + `spec.integration.failOpen: true`, Redis can degrade while the Pod-local L1 + remains usable; `false` makes Redis a readiness dependency. - The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); `EngineSelectorUnmatched` (Normal) when a configured selector first observes zero matching pods while engine pods are expected, transitions from matched to zero, or gains the diagnostic message during an upgrade from an older zero-count status. One advisory Event is recorded on the `CacheBackend` but triggered by engine-pod state rather than a CacheBackend condition transition: `InjectedEngineCrashLooping` (Warning) is emitted once when an injected engine pod's engine container is first observed in CrashLoopBackOff after connector injection — commonly a connector incompatibility (esp. a hybrid-attention model), surfaced as `EngineCompatibility=False/InjectedEngineCrashLooping`, but a crash-loop can also be a bad image/command/secret/OOM, so the cause is verified via the engine logs, not asserted by the Event. The controller does not watch engine pod status — it detects this on the next `CacheBackend` reconcile that lists the pods, so the Event reflects observation time, not the instant the container entered CrashLoopBackOff; a transient pod-list failure preserves the prior condition rather than re-firing it. - A `Normal InjectedByCacheBackend` Event is emitted on engine pods the mutating webhook stamps with both `inferencecache.io/injected-by` AND `inferencecache.io/injected-by-uid`, where the UID annotation matches the live CacheBackend's `metadata.uid` at reconcile time. The controller deliberately skips emission when (a) the named CR cannot be looked up (NotFound), (b) the UID annotation is absent (failurePolicy=Ignore forgery shape), or (c) the UID does not match the live CR (forgery or CR was recreated under the same name). Non-NotFound lookup errors surface as reconcile errors so controller-runtime retries with backoff. A pod explicitly opted out with a truthy `inferencecache.io/skip-inject` is instead stamped with `inferencecache.io/inject-skipped: skip-inject-annotation`; the same post-create controller emits a `Normal SkippedByOperator` Event only when both the truthy opt-out annotation and the webhook's skipped marker are present. The Events are recorded by a Pod-watching controller, not by the webhook itself: at mutating-admission time the apiserver hasn't assigned `metadata.uid` to the pod yet, so an event recorded from the webhook would carry `involvedObject.uid=""` and be invisible to describe (which filters events by UID). Routing the emission through a post-create controller is what guarantees the event reaches the user-visible surface. There is no `NoMatchingCacheBackend` Event; the no-match signals are `status.matchedEnginePods == 0`, `status.engineSelectorMessage`, and `EngineSelectorUnmatched` on the CacheBackend. -- Optional nested specs are pointer fields in Go so omitted objects stay absent in JSON and server-side apply does not claim empty nested objects. **`spec.integration` is the deliberate exception** — the defaulting webhook materialises it on admission, derives `engine` from canonical `spec.runtime` (or uses legacy `vllm`), and gives the nested schema-level defaults a parent object to apply to. The apiserver then applies the `+kubebuilder:default=` markers on `mode` (`Offload`), `role` (`ReadWrite`), `failOpen` (`true`), and `firstEventTimeout` (`5m`) before persisting the CR. Operators reading the persisted CR therefore see the effective compatibility fields explicitly. The `IntegrationFailOpen` reader helper still exists (nil spec or nil field ⇒ `true`) as defence-in-depth for callers that bypass admission. Other optional nested specs (`spec.autoscaling`, `spec.template`, `spec.engineSelector`) are NOT materialised — omitted means absent. Webhook-stamped and apiserver-stamped fields are owned by their respective field managers, not the operator's SSA apply, so SSA semantics for operator-set fields are unaffected. +- Optional nested specs are pointer fields in Go so omitted objects stay absent + in JSON. `spec.integration` and `spec.observation` are the deliberate + exceptions: the defaulting webhook materializes them so their nested defaults + persist. Read-time helpers retain defensive defaults for callers that bypass + admission. ## Admission @@ -744,44 +650,34 @@ The controller serves two webhooks for CacheBackend, both registered as `failure ### Defaulting (mutating) -Most Phase-1 literal defaults ride on `+kubebuilder:default=` markers stamped by the apiserver before the webhook runs (`spec.type=LMCache`, `spec.deploymentKind=Deployment`, `spec.replicas=1`, `spec.integration.mode=Offload`, `spec.integration.role=ReadWrite`, `spec.integration.failOpen=true`, `spec.observation.firstEventTimeout=5m`). The webhook handles context-dependent defaults; operator-set values are never clobbered. - -| Field | Default | Layer | -|---|---|---| -| `spec.type`, `spec.deploymentKind`, `spec.replicas`, `spec.integration.{mode,role,failOpen}`, `spec.observation.firstEventTimeout` | per-field literals (see field godoc) | `+kubebuilder:default=` markers — apiserver | -| `spec.observation.firstEventTimeout` (when `spec.observation` is omitted entirely) | `5m` | webhook materialises `spec.observation` so the nested marker has a parent object to apply to | -| `spec.autoscaling.minReplicas` (FIRST APPLY ONLY, when `spec.autoscaling != nil` and `spec.autoscaling.minReplicas == nil`) | `= spec.replicas` (post-marker-default; skipped when `spec.replicas` is 0 to avoid violating the schema's `Minimum=1`) | webhook | - -The `spec.autoscaling.minReplicas` default is **first-apply only**. The defaulter refuses to overwrite a non-nil value, AND once stamped the field is owned by the apiserver field manager, so a subsequent edit to `spec.replicas` does NOT recompute or move `minReplicas`. This matches the standard Kubernetes HPA convention that scaling intent flows through HPA fields once an HPA owns the workload — to widen or narrow the autoscaling band post-apply, edit `spec.autoscaling.minReplicas` directly. (The `replicas=0` + autoscaling + nil minReplicas case is rejected at admission rather than defaulted; see the validator table below.) +Schema defaults set `spec.type=LMCache`, +`spec.integration.mode=Offload`, `spec.integration.role=ReadWrite`, and +`spec.integration.failOpen=true`. The webhook materializes omitted +`integration` and `observation` parents and sets +`observation.firstEventTimeout=5m`. It does not default workload replicas, +autoscaling, provider images, or an engine image. ### Validating -Rejects structurally-broken specs that the reconciler cannot do anything useful with, with field-scoped error messages. Multiple violations on a single spec are aggregated into one `Invalid` status so kubectl prints them together. - -| Rule | Rejects | -|---|---| -| Cache hierarchy must be internally consistent | A provider-specific typed block does not match `remoteStorage.provider`/`ownership`, `lmCache` is used with a non-LMCache type, or host-only configuration requests workload autoscaling. | -| External remote storage requires an endpoint | `remoteStorage.ownership=External` without `remoteStorage.endpoint`; managed ownership rejects a user-supplied endpoint. | -| Engine wire must accept the provider binding | Every `(runtime, type)` adapter must explicitly implement the remote-binding contract, and admission rejects a binding it does not accept (`lm`, `resp`, `mooncakestore`, or host-only). Native SGLang HiCache accepts only the nil host-only binding; attaching any `remoteStorage` is rejected. | -| Provider resources must be valid | Typed provider resource blocks are checked for request/limit relationships, claims, quantities, resource names, extended resources, and hugepage alignment, with errors reported at the selected provider path. | -| Endpoint ownership is explicit | `spec.remoteStorage.endpoint` is required for External ownership and rejected for Managed ownership. A managed endpoint always comes from the live Service the controller provisions, so a user-supplied value would be misleading. Whitespace-only values are treated as empty. | -| Cross-namespace endpoint requires opt-in | `spec.remoteStorage.endpoint` resolves to a Service in a namespace other than the CacheBackend's, while `spec.allowCrossNamespace` is `false`. Crossing the namespace is a tenancy boundary the operator must acknowledge. Bare hostnames, IPs, and unqualified names pass through because no namespace can be inferred. | -| `spec.replicas=0` + autoscaling requires explicit `minReplicas` | `spec.replicas=0` with `spec.autoscaling != nil` and `spec.autoscaling.minReplicas == nil`. The defaulter declines to compute `minReplicas` from a 0 replicas value (it would violate the schema's `Minimum=1`), so without this rule the apiserver accepts the CR and the reconciler's HPA fallback silently picks `1` — overriding the operator's "scale to zero" intent with no notification. The rejection tells the operator to either set `minReplicas` explicitly or remove `spec.autoscaling` to scale to zero unconditionally. | -| `spec.integration.engineOverrides` cannot touch reserved args/env | An entry in `engineOverrides.args` / `engineOverrides.suppressArgs` matches a leading flag token the adapter declares as `ReservedArgs()`, or an entry in `engineOverrides.env` / `engineOverrides.suppressEnv` matches a name in `ReservedEnv()`. The rejection names both the offending flag/env and the adapter so the operator can fix the spec rather than wait for the engine to crash. The reserved set is per-adapter (the vLLM+LMCache adapter reserves `--kv-transfer-config`, `VLLM_USE_V1`, `LMCACHE_REMOTE_URL`, `INFERENCECACHE_FAIL_OPEN`, `PYTHONHASHSEED`). | -| Provider resource limits and requests must agree | Under `spec.remoteStorage..resources`, overcommittable resource limits must be ≥ requests; hugepages and extended resources must use equal request/limit values. | -| Requests-only is rejected for non-overcommittable resources | A hugepage or vendor-prefixed extended resource is present in a provider `resources.requests` map without a matching limit. | -| Provider `resources.claims` is not supported | A selected provider resource block contains Dynamic Resource Allocation claim names, but the renderer does not yet create matching pod-level `spec.resourceClaims`. | -| Extended-resource quantities must be integers | A selected provider resource block gives a vendor-prefixed extended resource a fractional value. | -| Hugepage quantities must align to the page size | A selected provider resource block contains a positive `hugepages-` quantity that is not a whole multiple of its page size. | -| Provider resource quantities must be non-negative | A selected provider `resources.requests` or `resources.limits` entry is negative. | -| Provider resource names must be valid | A selected provider resource key is not a valid standard, hugepage, or vendor-prefixed container resource name. | -| Runtime/cache pair must be supported by an installed adapter | The `(runtime, engine-cache type)` pair has no registered runtime adapter, so the reconciler cannot observe engine compatibility and the pod webhook would fail open without injecting engine config. The shipping pairs are `VLLM/LMCache`, `SGLang/LMCache`, and `SGLang/SGLangHiCache`; remote provider selection is validated independently through `remoteStorage`. The registry's `SupportedPairs` list is included in the field-scoped rejection. | -| Events-only requires `spec.type=LMCache` | `spec.integration.mode=EventsOnly` with any `spec.type` other than `LMCache` (the default). Events-only wires no KV connector, so declaring an offload-oriented cache type is contradictory. `LMCache` supplies the kvevent-subscriber that the routing tier needs. See [Events-only mode](#events-only-mode-specintegrationmode--eventsonly). | -| Events-only forbids `spec.autoscaling` | `spec.integration.mode=EventsOnly` with `spec.autoscaling` set. An events-only backend provisions no server workload, so there is nothing to autoscale. Field-scoped to `spec.autoscaling`. | - -The structural rules are an ordered, pluggable list (`CacheBackendValidator.Rules`); the runtime/backend compatibility check runs separately because it needs to consult the shared `adapterruntime.Registry` rather than just the spec. - -`ValidateUpdate` only rejects violations the update *introduces*: errors that already existed on the previous object are filtered out so an unrelated edit (a label tweak, an annotation) on a CR admitted under a laxer rule set is not suddenly un-updatable. A `kubectl edit` that flips a previously-valid field into an invalid one is still rejected, because the violation is then new to the diff. Errors are compared by `(Type, Field, BadValue, Detail)`, so an operator changing one bad endpoint to a different bad endpoint on the same field counts as a fresh violation — the rule still bites when the operator actively edits the bad field. +Validation aggregates field-scoped violations into one Kubernetes `Invalid` +response. The current rules enforce: + +- a typed LMCache `PodLocal` topology (NodeLocal is published but rejected + until Phase 8), a digest-pinned MP-server image, non-colliding ports, and + sufficient CPU/memory resources; +- Redis as the only remote provider, with explicit Managed/External ownership, + a valid External endpoint, provider/config agreement, and only RESP features + implemented by the pinned adapter; +- Kubernetes resource request/limit, name, quantity, hugepage, extended + resource, and unsupported-claim constraints; +- the shipping runtime/cache pairs and each adapter's accepted binding; +- EventsOnly and SGLangHiCache shape constraints; +- `ReadWrite` as the only LMCache role; +- valid kernel-check annotations; and +- protection of adapter-reserved engine arguments and environment variables. + +`ValidateUpdate` validates the new object. Delete is always allowed so an +operator can remove invalid state. ### Breaking API cleanup @@ -814,44 +710,17 @@ The CRD field default is byte-identical to the prior behavior: a CacheBackend wi #### Reserved declarations and admission hard-reject -Each `KVCacheRuntimeAdapter` declares two methods: - -- `ReservedArgs() []string` — leading flag tokens the user MUST NOT override or suppress. -- `ReservedEnv() []string` — env var names the user MUST NOT override or suppress. - -The validating webhook selects the adapter from `spec.runtime`, then iterates -its reserved lists and **hard-rejects** any `engineOverrides.{args,suppressArgs}` entry -that overlaps `ReservedArgs()` and any `engineOverrides.{env,suppressEnv}` -entry that overlaps `ReservedEnv()`. The rejection names the offending -flag/env and the adapter. Warning-only would let a user silently un-wire the -integration and discover it via a crashed engine; the hard-reject keeps the -breadcrumb at admission time. +Each runtime adapter declares `ReservedArgs()` and `ReservedEnv()`. Admission +rejects any override or suppression that overlaps those lists: -The legacy vLLM+LMCache adapter (`internal/adapters/builtin/runtime/vllm_lmcache.go`) reserves the args/env the integration cannot function without: - -- `ReservedArgs()`: `--kv-transfer-config` (the LMCache connector wiring). -- `ReservedEnv()`: `VLLM_USE_V1` (selects the engine codepath the connector targets), `LMCACHE_REMOTE_URL` (the resolved cache endpoint), `INFERENCECACHE_FAIL_OPEN` (mirror of `spec.integration.failOpen` — overriding it would silently desync the pod from the CR contract), `PYTHONHASHSEED` (pins the deterministic `NONE_HASH` so LMCache reload matches under TP>1 — overriding or suppressing it silently 0-hits reload). - -The same reserved set applies when a legacy vLLM/LMCache object has -an External LMCacheServer binding or a Mooncake binding: the selected runtime -adapter still runs the LMCache connector and varies only the structured -binding's protocol and endpoint. Admission therefore rejects -an override that would remove connector wiring regardless of provider -ownership. See -[Mooncake provider configuration](#mooncake-provider-configuration). - -The typed PodLocal vLLM MP adapter reserves a narrower and different set: -`ReservedArgs()` = `--kv-transfer-config`, -`--disable-hybrid-kv-cache-manager`; `ReservedEnv()` = `PYTHONHASHSEED`, -`INFERENCECACHE_FAIL_OPEN`. It does not inject or reserve -`LMCACHE_REMOTE_URL`, `VLLM_USE_V1`, or the legacy serde/local-CPU variables. - -The SGLang+LMCache adapter (`internal/adapters/builtin/runtime`) reserves a **different** set, because SGLang's engine-side wire is the LMCache MP wire, not the `lm://` one (see [SGLang engine support](#sglang-engine-support)): `ReservedArgs()` = `--enable-lmcache`, `--lmcache-config-file`; `ReservedEnv()` = `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN`. Suppressing `--lmcache-config-file` un-wires MP mode (the engine aborts at startup without it), hence its reservation. In MP mode the lm:// `LMCACHE_REMOTE_URL` is neither injected nor reserved, and `VLLM_USE_V1` / `PYTHONHASHSEED` are never injected for SGLang. Reservation is per-adapter precisely so each engine guards only the flags/env its own integration cannot function without. +| Adapter | Reserved args | Reserved env | +|---|---|---| +| vLLM typed PodLocal MP | `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager` | `PYTHONHASHSEED`, `INFERENCECACHE_FAIL_OPEN` | +| SGLang typed PodLocal MP | `--enable-lmcache`, `--lmcache-config-file`, `--enable-metrics` | `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN` | +| SGLangHiCache | its injected HiCache flags | none unless introduced by the adapter | -`LMCACHE_CHUNK_SIZE`, `LMCACHE_REMOTE_SERDE`, `LMCACHE_LOCAL_CPU`, and -`LMCACHE_MAX_LOCAL_CPU_SIZE` belong only to the legacy IP adapter. They remain -unreserved while that compatibility adapter exists, but current MP manifests -must use the typed `spec.lmCache` hierarchy instead. +The removed IP connector environment is neither injected nor part of the +current override contract. #### Shape rationale (A vs. B) @@ -860,7 +729,10 @@ Two shapes were on the table: - **A — typed K8s vocabulary** (`[]string` args, `[]corev1.EnvVar` env, plus suppression). Chosen. - **B — free-form magic keys** (`cpuMode: "true"`, `gpuLimit: "0"`, `extraArgs: "..."`). Rejected. -A is more general: Mooncake remote bindings, the SGLang adapter, and further engine/backend pairs plug in with no per-adapter free-form schema churn. It keeps the CRD disciplined. B is faster to ship but bakes engine-specific knobs into the CRD, which is the trap an "engine-agnostic backend" surface is meant to avoid. +A is more general: Redis bindings, both LMCache runtime adapters, and further +engine/backend pairs plug in with no per-adapter free-form schema churn. It keeps +the CRD disciplined. B is faster to ship but bakes engine-specific knobs into +the CRD, which is the trap an "engine-agnostic backend" surface is meant to avoid. #### Residual risk @@ -872,14 +744,14 @@ A user can still set non-reserved values that break the engine in subtle ways th ### Mutating Pod webhook (engine wiring) -A separate mutating admission webhook on `corev1/v1.Pod` (`name: mpod.inferencecache.io`) auto-wires user-supplied inference engine pods to the matching `CacheBackend` across all three lifecycle shapes: controller-managed server backends, operator-managed External endpoints, and engine-local backends such as SGLang HiCache. Operators do not have to hand-edit the adapter-specific args, env, sidecars, volumes, or mounts onto their pod templates. The handler lives in `internal/webhook/pod` and runs on every Pod CREATE. +A separate mutating admission webhook on `corev1/v1.Pod` (`name: mpod.inferencecache.io`) auto-wires user-supplied inference engine pods to the matching `CacheBackend` across managed Redis, external Redis, host-only MP, EventsOnly, and SGLang HiCache shapes. Operators do not have to hand-edit the adapter-specific args, env, sidecars, volumes, or mounts onto their pod templates. The handler lives in `internal/webhook/pod` and runs on every Pod CREATE. | Aspect | Behavior | |---|---| | Selection | Lists `CacheBackend`s in the pod's namespace via the manager's **APIReader** (uncached live client; an informer-cache miss on a freshly-Ready backend would leave the pod permanently unwired since pod CREATE is a one-shot), then matches `pod.Labels` against each `Spec.EngineSelector.MatchLabels`. The first matching `CacheBackend` wins; one with a nil or empty `EngineSelector` is skipped (a "match-everything" selector would silently claim every pod in the namespace). | -| Injection | Resolves the runtime adapter via `runtime.Registry.Select(runtimeID, cache)`, resolves `spec.remoteStorage` independently, and constructs a structured provider `Binding{Protocol, Endpoint}`. Managed ownership uses `status.endpoint` from the live Service; External ownership uses the trimmed, provider-validated `spec.remoteStorage.endpoint` with no fallback to stale status; omitted `remoteStorage` produces a nil host-only binding. `SupportsBinding` is part of the required runtime adapter interface, and the webhook passes the binding directly to `adapter.InjectEngineConfig`, so the adapter selects the LMCache, RESP, or Mooncake engine wire from the binding protocol instead of inferring storage from `spec.type`. A non-nil binding with a missing endpoint fails open. Events-only skips engine injection because it wires no KV connector and appends only the kvevent-subscriber sidecar. Adapters preserve existing user args/env and make repeat injection idempotent. | +| Injection | Resolves the runtime adapter via `runtime.Registry.Select(runtimeID, cache)`, resolves `spec.remoteStorage` independently, and constructs a structured provider `Binding{Protocol, Endpoint}`. Managed ownership uses `status.remoteStorage.endpoint` from the live Service; External ownership uses the trimmed, provider-validated `spec.remoteStorage.endpoint` with no fallback to stale status; omitted `remoteStorage` produces a nil host-only binding. `SupportsBinding` is part of the required runtime adapter interface, and the webhook passes the binding directly to `adapter.InjectEngineConfig`, so the adapter selects host-only MP or the RESP wire from the binding protocol instead of inferring storage from `spec.type`. A non-nil binding with a missing endpoint fails open. Events-only skips engine injection because it wires no KV connector and appends only the kvevent-subscriber sidecar. Adapters preserve existing user args/env and make repeat injection idempotent. | | Annotations | Stamps TWO annotations on every successfully mutated pod: `inferencecache.io/injected-by: /` (operator-readable identity, shows in `kubectl describe pod`) AND `inferencecache.io/injected-by-uid: ` (the matched CR's metadata.uid). Successful injection also clears any stale `inferencecache.io/inject-skipped` marker. Reads `inferencecache.io/skip-inject: ` as an opt-out: the webhook returns Allowed, skips engine wiring, clears any stale injected-by/injected-by-uid pair, and stamps `inferencecache.io/inject-skipped: skip-inject-annotation` so explicit operator opt-out is distinguishable from selector drift. On all other fail-open returns after the pod is decoded (list/no match/missing endpoint/adapter errors), the webhook strips stale injected-by/injected-by-uid and inject-skipped annotations so a user cannot trick the events controller by pre-stamping a pod template. Decode failures fail open before a Pod exists to patch, so stale annotations cannot be cleared on that path. | | Events | The webhook itself does NOT record events (the apiserver assigns `metadata.uid` after mutating admission, so a webhook-recorded event would carry `involvedObject.uid=""` and be invisible to `kubectl describe pod`). Instead, the pod-watching `engine-pod-events` controller reads the persisted decision annotations after CREATE. For injected pods, it validates `inferencecache.io/injected-by-uid` against the live CR's `metadata.uid` and records a `Normal InjectedByCacheBackend` event on the now-persisted pod. For explicitly skipped pods carrying both a truthy `inferencecache.io/skip-inject` and `inferencecache.io/inject-skipped: skip-inject-annotation`, it records a `Normal SkippedByOperator` event on that pod. The skip marker is not authenticated, and `skipInjection` treats a pre-existing correct marker as already converged; `SkippedByOperator` therefore means the persisted pod carries the explicit opt-out plus skipped marker, not proof that the webhook authored the marker. The UID match REDUCES — but does NOT eliminate — the failurePolicy=Ignore forgery surface for injected pods: a casual copy-paste of an injected pod's annotations into a fresh template won't match the live CR's UID, but `metadata.uid` is not secret, so a pod creator with `get` RBAC on CacheBackends can read it and stamp the pair correctly. The injected Event signals "the webhook claims this pod was injected and the claim is consistent with the live CR," not "the webhook was cryptographically authenticated." The controller skips the injected event when the CR is missing, the UID annotation is absent, or the UID does not match — see the controller godoc for the full skip table. controller-runtime's EventBroadcaster aggregates duplicates on the apiserver side, so a re-enqueue across controller restarts upserts the existing event rather than spamming. | | Idempotency | The handler calls the adapter unconditionally on every admission and trusts the adapter to converge the full injected contract. For LMCache this is env plus the engine-specific required surface — `--kv-transfer-config` for vLLM; for SGLang `--enable-lmcache` + `--lmcache-config-file` **plus** the MP-worker native sidecar and the shared config / `/dev/shm` volumes + mounts. Its merge primitives (`upsertEnv` / `upsertArgPair` / `upsertFlag`, and for SGLang `adoptContainer` / `adoptVolume` / `upsertMountByName`) converge on the desired value rather than appending a duplicate. The SGLang `adopt*` pair additionally distinguishes the adapter's own prior injection (converge) from an operator's object squatting a reserved name (reject → fail-open admit) — see [Names the MP wire reserves](#sglang-engine-support). Native HiCache validates all reserved arguments against the original pod before mutation, preserves one matching or well-formed operator-supplied value, appends each missing canonical argument once, and rejects conflicts, malformed values, or duplicates without partially changing the pod. Re-admission of a fully-injected pod therefore produces an empty JSON-patch set. Trusting the adapter rather than a handler-side env-presence shortcut avoids the trap where a partially-injected pod is admitted permanently missing the rest of the contract. | -| Fail-open | Every error path (decode failure, list error, no matching backend, missing `status.endpoint`, no registered adapter, adapter rejection, re-encode failure) returns `admission.Allowed(...)` with a reason — webhook errors MUST NOT block engine admission. `MutatingWebhookConfiguration.failurePolicy` is also pinned to `Ignore` as a belt-and-suspenders second layer. | +| Fail-open | Every error path (decode failure, list error, no matching backend, missing managed `status.remoteStorage.endpoint`, no registered adapter, adapter rejection, re-encode failure) returns `admission.Allowed(...)` with a reason — webhook errors MUST NOT block engine admission. `MutatingWebhookConfiguration.failurePolicy` is also pinned to `Ignore` as a belt-and-suspenders second layer. | | Verbs | `CREATE` only. UPDATE re-admissions to a running pod don't re-inject (and the engine container can't pick up env changes without a restart anyway); UPDATEs to engine pods are rare in this fleet. | diff --git a/docs/design/crd-contract.md b/docs/design/crd-contract.md index 86af8108..69883347 100644 --- a/docs/design/crd-contract.md +++ b/docs/design/crd-contract.md @@ -47,7 +47,7 @@ Every **spec-reconciled** CRD that carries a status follows the same three rules | Status | Field(s) | Writer | |---|---|---| -| `CacheBackend.status` | `matchedEnginePods`, `engineSelectorMessage`, `conditions`, `firstKVEventObservedAt`, `firstAvailableAt`, `observedServerInstance`, `endpoint`, `failOpen`, `observedGeneration` | CacheBackend reconciler | +| `CacheBackend.status` | `connector`, `remoteStorage`, `matchedEnginePods`, `engineSelectorMessage`, `conditions`, `firstKVEventObservedAt`, `firstAvailableAt`, `failOpen`, `observedGeneration` | CacheBackend reconciler | | `CacheBackend.status` | `indexParticipation` | CacheIndex snapshot poller | | `CachePolicy.status` | `conditions`, `observedGeneration` (reserved) | — (see note) | | `CacheTenant.status` | `indexEntries`, `conditions`, `observedGeneration` | CacheIndex snapshot poller (per-tenant projection of `/snapshot`) | diff --git a/docs/design/kvevent-subscriber-wiring.md b/docs/design/kvevent-subscriber-wiring.md index 1cd2edb9..3aad8b14 100644 --- a/docs/design/kvevent-subscriber-wiring.md +++ b/docs/design/kvevent-subscriber-wiring.md @@ -43,18 +43,17 @@ to turn it on.** Concretely: * `KVCacheRuntimeAdapter` gains `ObservationSidecar(cb, pod) (*corev1.Container, error)`. - The vLLM/LMCache, vLLM/Mooncake, SGLang/LMCache, and SGLang/HiCache adapters return the `kvevent-subscriber` + The vLLM/LMCache, SGLang/LMCache, and SGLang/HiCache adapters return the `kvevent-subscriber` container spec (via their shared internal subscriber renderer — the KV-event stream is the engine's own ZMQ publisher, independent of the L2 store; each adapter pins its engine's `--hash-scheme` tag + ZMQ port); the reference adapter returns `(nil, nil)`. External Redis ownership stays on the runtime/cache adapter and can attach - observation. Legacy IP/Mooncake adapters retain observation behavior only - until Phase 7. + observation. The former IP adapters were removed in Phase 7. * The Pod webhook (`internal/webhook/pod/podinjector.go`) calls `ObservationSidecar` right after `InjectEngineConfig`. A non-nil container is appended to `pod.Spec.Containers` (idempotent — skipped if a container by the well-known name is already present). Errors fail open, matching the rest of the webhook. -* **The vLLM/LMCache, vLLM/Mooncake, SGLang/LMCache, and SGLang/HiCache adapters return nil unless the +* **The vLLM/LMCache, SGLang/LMCache, and SGLang/HiCache adapters return nil unless the controller's `--kvevent-subscriber-image` flag is set** (all go through the same shared internal renderer, so the opt-in behaviour is identical). An unconfigured image would put the sidecar container into `ImagePullBackOff`, which keeps the engine pod from going Ready — the @@ -96,9 +95,7 @@ Concretely: The SGLang adapter reuses the same shared subscriber, only its `--hash-scheme` tag differs (SGLang adopted vLLM's ZMQ KV-event wire); the seam is what would let a genuinely different future engine return a different sidecar (e.g. a different ZMQ port or a - completely different observation mechanism). A Mooncake remote binding uses the same - vLLM kvevent-subscriber because the engine is still vLLM and its KV events still come - from vLLM's ZMQ publisher (scheme-tagged `vllm`); only the backend store differs. A + completely different observation mechanism). A future backend that fronts a non-vLLM engine, or exposes observation data some other way, could still return `nil` or a different container here. **DaemonSet remains an option for any future adapter** that wants it — it just isn't this PR. diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 3e8b668f..764fb385 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -1,6 +1,6 @@ # Design Roadmap: LMCache Multiprocess Migration -Status: **Phases 0–4 complete (2026-08-11)** · Scope: +Status: **Phases 0–5 and 7 complete; Phase 6 was not required (2026-08-11)** · Scope: deprecate and remove this project's LMCache in-process data plane, converge vLLM and SGLang on LMCache multiprocess (MP) mode, and model Pod-local and node-local MP server placement without conflating either with @@ -104,7 +104,11 @@ code lands. | D11 | Each supported vLLM integration explicitly identifies its MP connector implementation; the initial reference baseline uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial adapter uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future adapter revision may validate a different implementation explicitly. | | D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. The selected adapter renders its engine-specific connector contract, while normal engine initialization is the authoritative compatibility check; tested images are neither an admission allowlist nor a mutation default. | -## Current state +## Migration baseline (before Phase 1) + +This table records the implementation state that motivated the roadmap. It is +historical, not the current production contract; Phase completion and the +current MP-only contract are tracked below. | Area | Current behavior | Gap to target | |---|---|---| @@ -362,7 +366,7 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 4 | vLLM PodLocal MP | Phase 3 | complete | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | complete | | 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 and Phase 5 findings | -| 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | not started | +| 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | in progress | | 8 | NodeLocal shared MP server topology | Phases 3–4; does not block Phase 7 | not started | ## Phase 0 — design freeze and compatibility baseline @@ -680,88 +684,69 @@ probe, so traffic tests waited for the engine health endpoint. ### Objective -Convert repository-owned consumers to MP without silently changing cross-Pod -sharing or remote-L3 semantics. +Convert every repository-owned LMCache consumer to typed MP without silently +changing cross-Pod sharing or remote-L3 semantics. ### Scope -Repository migration is required. Inventory/migration tooling is conditional -because Phase 0 found no external users or installed legacy objects. - -### Deliverables +Includes samples, reference manifests, support tables, CLI/docs, and ordinary +test fixtures. Legacy implementation and intentional compatibility coverage +remain until Phase 7. Physical removal is outside this phase. -- [x] Convert canonical samples, reference-stack manifests, support tables, CLI - output, documentation, screenshots, and non-transition fixtures to MP. -- [x] Remove language that presents the legacy LMCache server as a CPU profile - or default backend. -- [x] Reconfirm the zero-external-consumer assumption before removal, within the - evidence boundary recorded below. -- [x] Re-evaluate the conditional tooling trigger. No external consumer or - installed legacy-object evidence appeared, so migration tooling, - deprecation Events, and a compatibility gate were not activated. +The Phase 0 owner audit and Phase 5 repository inventory found zero external +consumers and zero installed legacy objects. This evidence covers this +repository and the recorded owner audit, not every organization source or OCI +cluster. Because no input population appeared, migration tooling and Phase 6 +were not activated. -### Phase 5 inventory and disposition +LMCacheServer and Mooncake are never translated to Redis automatically; the +operator must choose host-only MP, Redis, or a future typed adapter. +`remoteSerde` has no generic replacement and is removed unless a future typed +adapter validates equivalent semantics. -The repository-wide `rg` inventory was classified before editing: +### Deliverables -| Class | Findings | Disposition | -|---|---|---| -| Production/current consumers | Legacy LMCache samples (`cachebackend-lmcache*`, External, CPU override, paired/override), five recipes, flat SGLang samples, vLLM/SGLang reference manifests, quickstart/concepts/site pages, support tables, and the reference Helm values file. | Converted to typed PodLocal MP with explicit host-only or Redis semantics. The unvalidated Helm mapping, legacy CPU-only LMCache sample, and Mooncake sample were removed rather than translated inaccurately. | -| Legacy implementation for Phase 7 | Topology-less API fields/provider enums and CRD schema, LMCacheServer/Mooncake renderers, vLLM IP connector/wire helpers, endpoint parser, lifecycle/status code, and the doctor endpoint-scheme parser. | Retained unchanged so legacy alpha objects remain reconcilable until Phase 7. | -| Historical/migration documentation | This roadmap, the SGLang MP spike, LMCache-server persistence decision, and legacy portions of the API design. | Retained with explicit history/compatibility banners; current sections and links point to typed MP. | -| Intentional compatibility tests | Go tests for legacy render/admission behavior; C2/C6 scripts/workflows; legacy portions of default-install smoke. | Retained for Phase 7 safety, labelled legacy-only. C2/C6 scheduled triggers were removed; default-install smoke uses inline legacy fixtures instead of current samples. | - -No repository-owned screenshot asset contained a legacy deployment. CLI golden -output contained no legacy backend recommendation, so no output fixture changed; -the `doctor` `lm://` parsing branch is implementation compatibility for Phase 7. - -**External-consumer evidence boundary.** The Phase 5 repository inventory found -no cross-repository manifest, API client, generated consumer, or migration input, -and the Phase 0 owner audit remains zero for external consumers and installed -legacy objects. This phase did not query every OCI cluster or organization-wide -source repository, so the zero claim is limited to the repository evidence and -the recorded owner/Phase 0 confirmation. No contrary evidence appeared; adding -tooling without an input population would therefore create an unused migration -surface. - -Migration rules: - -| Existing object | Automatic portion | Required operator choice | -|---|---|---| -| SGLang MP with flat worker fields | Move image, port, and host-memory capacity into `lmCache.podLocal.server` and set `lmCache.topology: PodLocal`. | Confirm pinned image/resources and supported Kubernetes version. | -| vLLM IP host-only | Move host-memory capacity to PodLocal MP L1; select vLLM MP wire. | Confirm sidecar resources and accept the process/topology change. | -| vLLM IP + managed/external LMCacheServer | Preserve local capacity; remove `lm://`. | Select no L3 and lose cross-Pod sharing, or explicitly select a supported Redis/other L3. Never choose automatically. | -| vLLM IP + existing engine-side Mooncake provider | Preserve local intent only. | Wait for MP + Mooncake Store L3 support or migrate explicitly to Redis; URL config is not equivalent to MP adapter config. | -| Any IP object with `remoteSerde` | None. | Remove it or map it to a future typed L3 serde only when that adapter supports and validates the same semantics. | +- [x] Classify repository references as current consumers, Phase 7 legacy + implementation, history, or intentional compatibility coverage. +- [x] Convert current samples and manifests to typed `PodLocal` MP with explicit + host-only or Redis semantics. +- [x] Convert current documentation, support tables, CLI guidance, and ordinary + fixtures to typed MP. +- [x] Remove the unvalidated Helm mapping, legacy CPU-only LMCache sample, and + Mooncake sample instead of inventing unsafe translations. +- [x] Stop presenting the legacy LMCache server as a default backend or CPU + profile. +- [x] Retain the legacy implementation only for Phase 7 and clearly label all + history and compatibility coverage. +- [x] Reconfirm the zero-consumer finding and leave conditional migration + tooling inactive. ### Validation -- [x] Repository search finds no repository-owned production LMCache workload - still using IP, `lm://`, `LMCacheServer`, or flat SGLang MP fields. -- [x] `make verify-samples` admits every applicable migrated sample (25 passed, - 2 pre-existing explicit opt-outs, 0 failed); reference YAML parses, and - the typed vLLM/SGLang default-install smoke fixtures remain the current - admission path. The live kind default-install workflow was not run locally. -- [x] Every retained legacy reference is classified as Phase 7 implementation, - history/migration documentation, or intentional compatibility coverage. +Validation completed on 2026-08-11: + +| Item | Evidence | +|---|---| +| Repository inventory | No additional repository-owned/generated consumer, migration input, legacy screenshot, or CLI recommendation was found. | +| Production search | No current manifest retained `LMCacheConnectorV1`, `lm://`, `LMCacheServer`, IP wiring, or flat SGLang MP fields. | +| Automated tests | `git diff --check`, `go test ./...`, `make verify-samples` (25 passed, 2 explicit opt-outs), reference YAML parsing, shell checks, and `make ci` passed. | +| Optional check | The Python golden-vector check skipped because `xxhash` was unavailable; `make ci` still passed. | +| Environment | No Kubernetes cluster or GPU was needed or used. The live kind workflow was not run locally in this phase. | + +- [x] Every current repository consumer uses typed MP. +- [x] Ambiguous remote-storage examples were removed or require an explicit + operator choice; none was silently mapped to Redis. +- [x] Every remaining legacy reference is Phase 7 implementation, explicit + history, or intentional compatibility coverage. +- [x] The zero-consumer assumption was rechecked within its stated evidence + boundary. ### Exit criteria -- [x] Every repository-owned production/current LMCache workload uses typed MP. -- [x] No migration silently changes cross-Pod sharing behavior: each converted - object explicitly selects host-only or Redis, and ambiguous legacy - LMCacheServer/Mooncake examples were not auto-mapped. -- [x] Conditional tooling was not activated because the re-audit found no input - population or unknown legacy shape. - -Validation completed on 2026-08-11: `git diff --check`, `go test ./...`, -`make verify-samples`, shell syntax checks for the modified canaries/smoke, -reference-manifest YAML parsing, production/current negative searches, and -`make ci` all passed. `make ci` reported its optional golden-vector check as -skipped because the local Python environment lacked `xxhash`; the target itself -completed successfully. No Kubernetes cluster or GPU was required or used for -this repository-consumer migration, and the live kind default-install workflow -was not run locally. +- [x] Repository-owned production/current consumers use typed MP only. +- [x] Migration preserves explicit host-only versus shared-L3 intent. +- [x] Conditional tooling remains inactive because the audited input population + is zero. ## Phase 6 — reject new IP objects @@ -801,55 +786,86 @@ Otherwise this phase adds a temporary compatibility gate, not new IP features. ## Phase 7 — remove IP and the legacy LMCache server -- **Status:** Not started +- **Status:** Complete after QA review and revalidation (2026-08-11) - **Depends on:** Phase 5; Phase 6 only if activated ### Objective -Delete the IP data plane and all code/schema that exists only to support it. +Delete the legacy IP data plane and every production API/code path that exists +only to support it. LMCache selects typed MP only after this phase. ### Scope -Includes runtime adapters, provider protocols, controller workloads, status, -samples, tests, and legacy API fields. Historical migration documentation may -remain when clearly marked. +Includes runtime adapters, engine wire, provider lifecycle, API/schema, status, +metrics, Events, tests, samples, and documentation. Clearly marked history and +negative assertions may remain; compatibility implementation may not. + +Redis is the only current typed remote L3. Host-only MP creates no provider +workload, and managed Redis is a fixed standalone singleton. Useful managed +provider Pod scheduling/security fields live under +`spec.remoteStorage.workload`; generic replicas, autoscaling, deployment kind, +and legacy top-level template fields are removed. + +Mooncake remains future typed MP L2 work. Managed backend clusters, NodeLocal, +directional roles, SGLang TP>1, distributed execution, MLA, and robust MP-server +re-registration are outside this phase. None restores the old IP wire. +Inference-cache does not replace engine images; engine startup remains the +connector/package compatibility verdict. ### Deliverables -- [ ] Remove the vLLM legacy LMCache adapter. -- [ ] Remove `LMCacheConnectorV1` rendering. -- [ ] Remove `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, and other IP-only - injected settings. -- [ ] Remove `ProtocolLMCache` and the `lm://` endpoint parser/binding. -- [ ] Remove the managed and external `LMCacheServer` provider surface. -- [ ] Remove the standalone LMCache-server workload renderer. -- [ ] Remove IP-only status fields, metrics, Events, samples, and tests. -- [ ] Remove compatibility defaulting/validation and migration-only code after - any supported migration window closes. -- [ ] Remove legacy flat LMCache fields after their replacement is complete. -- [ ] Remove `LMCacheServer` from CRD enums and provider-specific schema. -- [ ] Remove or relocate top-level managed-provider workload fields according to - the Phase 0 decision. -- [ ] Regenerate CRDs, deepcopy code, examples, and reference documentation. +- [x] Delete the vLLM IP adapter, SGLang legacy wire helpers, + `LMCacheConnectorV1`, `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, + `ProtocolLMCache`, and the `lm://` parser/binding. +- [x] Delete managed/external LMCacheServer, the IP-wired Mooncake + implementation, standalone LMCache-server workloads, restart cascade, and + IP endpoint lifecycle/status behavior. +- [x] Remove flat LMCache fields, legacy provider schemas/enums, IP-only + status/metrics/Events, and compatibility defaulting/validation. +- [x] Move managed-provider scheduling/security to + `spec.remoteStorage.workload`; reject it for External ownership and remove + generic scaling/deployment fields. +- [x] Remove legacy canaries and compatibility fixtures; retain only explicit + history and negative assertions. +- [x] Regenerate CRDs and deepcopy code, and update current samples and docs. +- [x] Reconfirm zero consumers within the Phase 5 evidence boundary and skip + Phase 6/migration tooling. ### Validation -- [ ] `go test ./...` passes. -- [ ] `make verify-samples` passes. -- [ ] Default-install and upgrade smoke pass. -- [ ] Repository search finds no production-code references to: - - `LMCacheConnectorV1`; - - `LMCACHE_REMOTE_URL`; - - `ProtocolLMCache`; - - `lm://`; - - the managed `LMCacheServer` provider. -- [ ] Any retained historical reference is clearly marked as removed behavior. +Repository and CPU-only validation completed on 2026-08-11; post-QA GPU +regression ran in SJC dev on 2026-08-11 PDT (2026-08-12 UTC): + +| Item | Evidence | +|---|---| +| Repository gates | `git diff --check`, `go test ./...`, `make verify-samples` (25 passed, 1 explicit skip), generated-code checks, production searches, `make ci`, and `make cover-check` (90.1%) passed. | +| Fresh install | A kind smoke verified the MP-only schema, real Pod admission, managed Redis lifecycle/workload propagation, current samples, doctor, server surfaces, and idempotent re-apply. | +| Phase 5 upgrade | A separate kind smoke installed commit `10178558bfca308ee3a4b0d584efe4ed3b91197d`, created typed host-only and managed-Redis objects, upgraded to Phase 7, and preserved identity, topology, reconciliation, and Pod admission. | +| GPU environment | Kubernetes 1.31.1; one A100-SXM4-80GB per engine; LMCache 0.5.3 CUDA 12.9 client wheel; standalone sidecar `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b`; temporary Phase 7 controller `sha256:cd5c4da653bc5a8581e75f9e0668a103f920bc88fd230ce47b1aea6f6f90efc5`. | +| vLLM TP=1 | Engine `sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (0.25.1). A 910-token prompt stored 768 tokens; after `/reset_prefix_cache`, the same request retrieved 768 from MP L1. `Ready`, `ConnectorReady`, and `EngineKernelsHealthy` were True. | +| SGLang TP=1 | Engine `sha256:920df39109c60429b0a23eaacfd2786fcf1595c12f3ca4fc6e153b2abe34865f` (0.5.13.post1). A 1,091-token prompt stored 1,024 tokens; after `/flush_cache`, the same request reported 1,024 host-cached tokens and the server logged a 1,024-token retrieve. `Ready` and `ConnectorReady` were True. | +| Cleanup and limits | Test objects were deleted and the SJC control plane restored. The regression covers steady-state PodLocal only, not SGLang TP>1, NodeLocal, server restart/re-registration, or remote L3. | + +Both engines used a test-only init-container/shared-volume overlay for the +checksummed LMCache wheel. This is validation scaffolding, not the production +engine-image installation model. The optional Python golden-vector check +skipped because `xxhash` was unavailable; `make ci` still passed. + +- [x] Production search finds no current `LMCacheConnectorV1`, + `LMCACHE_REMOTE_URL`, `ProtocolLMCache`, `lm://`, or managed LMCacheServer + implementation. +- [x] Fresh-install and real Phase 5 typed-object upgrade smokes pass. +- [x] vLLM and SGLang TP=1 store → engine-cache reset → MP L1 retrieve pass on + GPU. +- [x] Retained legacy terms are clearly marked history or negative assertions. ### Exit criteria -- [ ] Only MP adapters can be selected for `spec.type: LMCache`. -- [ ] No controller workload or engine wire implements IP. -- [ ] No supported stored object requires the legacy schema. +- [x] `spec.type: LMCache` selects typed MP adapters only. +- [x] No controller workload, engine wire, served schema, or current manifest + implements the legacy IP path. +- [x] No supported stored object requires the removed schema. +- [x] Repository, upgrade, and required PodLocal GPU regressions pass. ## Phase 8 — NodeLocal shared MP servers @@ -927,6 +943,42 @@ exit criteria and do not block migration away from the legacy IP data plane: - [ ] Add each profile to the supported validation matrix only after its own GPU correctness, failure-recovery, and operability gates pass. +### Typed MP Mooncake L2 adapter + +Mooncake remains a supported provider direction, but its removed implementation +was coupled to the legacy IP connector and is not safe to restore. Future work +must add a new typed MP binding using LMCache's `mooncake_store` L2 adapter: + +- [ ] Add a provider-specific typed configuration for Mooncake metadata/master + addresses, protocol, segment sizing, local buffer sizing, credentials, + networking, and managed-versus-external lifecycle. +- [ ] Render `--l2-adapter` configuration through the common MP server without + exposing `lm://` or `LMCacheConnectorV1`. +- [ ] Define provider-scoped host-network/RDMA placement and security; do not + reuse an engine-global hostNetwork toggle. +- [ ] Validate cross-Pod sharing, restart/re-registration, failure isolation, + and both vLLM and SGLang client paths against pinned released artifacts. +- [ ] Never translate a legacy Mooncake object to Redis or infer typed adapter + settings from its old URL; migration requires an explicit operator choice. + +### Managed backend clusters + +The current managed Redis renderer intentionally creates one standalone Redis +Pod. Multiple replicas behind its Service would be independent keyspaces, not a +cluster. A future managed backend-cluster capability must therefore be +provider-specific: + +- [ ] Define Redis topology explicitly (for example standalone versus cluster), + including shard count, replicas per shard, stable identity, discovery, + failover, resharding, persistence, and readiness semantics. +- [ ] Decide whether inference-cache owns those resources directly or composes + with a dedicated Redis operator; keep the core runtime/provider boundary + inference-system-neutral. +- [ ] Verify that the selected LMCache RESP adapter or proxy endpoint supports + the advertised cluster behavior before exposing it in the support matrix. +- [ ] Keep generic `remoteStorage.workload` limited to Pod scheduling/security; + do not add replicas or autoscaling that silently changes provider semantics. + ### Directional LMCache roles for PD separation `ReadOnly` / `WriteOnly` remain generic CacheBackend API concepts, but all @@ -1105,17 +1157,17 @@ dated closure sections or separate phase documents. The migration is complete only when all of the following are true: -- [ ] `spec.type: LMCache` selects only MP implementations. +- [x] `spec.type: LMCache` selects only MP implementations. - [x] Both SGLang and vLLM pass the required PodLocal GPU matrix. - [x] Host-only MP is supported for both engines; optional L3 implementations are validated and versioned independently from the engine connector gate. - [x] Current MP server health is observable and steady-state cache behavior is tested. -- [ ] `remoteStorage` is optional L3 and no longer contains LMCacheServer. -- [ ] No production code injects `LMCacheConnectorV1`, `lm://`, or +- [x] `remoteStorage` is optional L3 and no longer contains LMCacheServer. +- [x] No production code injects `LMCacheConnectorV1`, `lm://`, or `LMCACHE_REMOTE_URL`. - [x] Remote-L3 lifecycle events do not automatically roll MP engines. -- [ ] Every old IP object has been migrated or intentionally deleted. +- [x] Every old IP object has been migrated or intentionally deleted. - [x] Canonical samples, reference manifests, CLI output, and design documents describe only the implemented MP behavior. - [x] NodeLocal, if enabled, guarantees same-node server selection and accurate diff --git a/docs/design/sglang-lmcache-mp-mode.md b/docs/design/sglang-lmcache-mp-mode.md index ef8f24ce..ac9fb66b 100644 --- a/docs/design/sglang-lmcache-mp-mode.md +++ b/docs/design/sglang-lmcache-mp-mode.md @@ -1,68 +1,30 @@ -# Design: LMCache MP mode — the converged worker model (SGLang now, vLLM migration) - -Status: **implemented and GPU-validated** for SGLang (Phase 2, increments 1–2); increment 3 (operator surface + the remaining SPOF containment) is open — see [Phased delivery](#phased-delivery). Facts below are live-validated unless marked otherwise. · Supersedes the "mirror the vLLM+LMCache adapter" model in [cachebackend-api.md](cachebackend-api.md) SGLang section · Built-in adapters: `internal/adapters/builtin/runtime`; public contract: `pkg/adapters/runtime` - -> **Implementation history, superseded for the current operator contract.** This -> document records the SGLang spike that established MP viability. Its flat -> worker fields, engine-image worker default, and predictions that vLLM MP was -> future work are historical. Current production behavior is the typed PodLocal -> API and common standalone-server renderer defined by +# Historical design: the SGLang LMCache MP spike + +Status: **historical implementation record**. Its spike evidence remains useful, +but its API shape and rollout predictions are superseded by the completed typed +MP migration. The current contract is in +[`cachebackend-api.md`](cachebackend-api.md). + +> **Everything below the current-state summary is implementation history, not +> an operator guide.** This document records the SGLang spike that established +> MP viability. Its flat worker fields, engine-image worker default, vLLM IP +> coexistence, and predictions that vLLM MP was future work are historical and +> were physically removed in migration Phase 7. Current production behavior is +> the typed PodLocal API and common MP-server sidecar renderer defined by > [`lmcache-multiprocess-migration-roadmap.md`](lmcache-multiprocess-migration-roadmap.md) > and [`cachebackend-api.md`](cachebackend-api.md). -**LMCache upstream now recommends multiprocess (MP) mode for *both* vLLM and -SGLang** (its quickstart: MP is *"recommended"* for vLLM via `LMCacheMPConnector`, -and *"the SGLang integration now defaults to MP mode"*). MP mode is a **node-local -`lmcache` worker** the engine attaches to over ZMQ + shared memory — not the -`lm://` standalone-remote-server model. This doc adopts MP as the **converged -worker model for both engines** and specifies the shared infrastructure (a -node-local worker + config-file wire + a shared L2 store) that carries it. - -The shipped `(sglang, LMCache)` adapter was built by analogy to the vLLM `lm://` -model, and **live GPU validation showed that is wrong for SGLang** — SGLang has no -`lm://` path at all, only MP. So SGLang is the first concrete implementation of the -converged model and the driver for this design; the vLLM migration reuses the same -infrastructure and is future work (see [Support matrix](#converged-foundation-mp-for-both-engines)). - -**Status: implemented and GPU-validated (Phase 2, increments 1–2).** The wire -described here is what the adapter renders today. The advisory admission warning -that used to flag the SGLang pair as non-functional (`sglangLMCacheDataPlaneWarning`) -is **retired** — it existed only because the shipped `lm://` wiring cached nothing, -which this design replaced. What remains open is tracked in -[Phased delivery](#phased-delivery). - -> **API ownership update.** The MP worker is still the validated engine-side -> wire, but Redis is no longer selected by that runtime adapter. Canonical -> resources use `spec.runtime: SGLang`, `spec.type: LMCache`, and optionally -> `spec.remoteStorage{provider: Redis, ownership: Managed}`. Without -> `remoteStorage`, the same worker starts without `--l2-adapter` and provides a -> host-only LMCache tier. The storage-provider registry owns Redis rendering and -> gives the engine adapter an optional RESP binding. Historical descriptions of -> `ResolveCacheServer` below document the pre-separation implementation. - -## TL;DR - -- Both engines can drive LMCache in **multiprocess (MP) mode** (upstream- - recommended): the engine attaches to a **node-local `lmcache` worker** over ZMQ - (`mp_host`/`mp_port`) + a shared-memory data path. SGLang configures it via a - **`--lmcache-config-file`** (the injected remote-connection/tuning `LMCACHE_*` - env is ignored; `LMCACHE_USE_EXPERIMENTAL` gates the connector); vLLM via - `LMCacheMPConnector` + `kv_connector_extra_config`. **SGLang is MP-only** (no - `lm://` path exists); vLLM keeps its `lm://` path too — see the support matrix - below. -- Cross-node KV sharing is a **networked L2 store behind the MP worker** - (`--l2-adapter` = `resp`/Redis, `s3`, `mooncake_store`, or `p2p`) — **not** the - `lm://` server, which is not even a valid MP `--l2-adapter` type. -- Runtime and provider capabilities resolve independently: the Managed Redis - provider renders the shared L2 and produces a RESP binding; the SGLang - runtime adapter accepts that binding or nil for host-only operation. - Engine injection adds a **node-local MP-worker native sidecar (which writes - the config file it then serves) + the engine wire** to the engine pod. - `mp_host=127.0.0.1` (worker co-located in - the pod), so — unlike Mooncake — the engine needs **no `hostNetwork`**. The - packaging question (a **GPU-less sidecar** vs. a single container) is **resolved - in favour of the GPU-less native sidecar** — spiked, then GPU-validated end to end; - see [Resolved: GPU-less sidecar](#resolved-gpu-less-sidecar-vs-same-container). +## Current-state summary + +- vLLM and SGLang support only typed PodLocal LMCache MP in the current API. +- Admission injects a CacheBackend-configured `lmcache-mp-server` native sidecar; + it does not own or replace the engine image. +- Omitting `remoteStorage` selects host-only MP. Explicit Redis selects the only + currently supported remote L3. Removed LMCacheServer and legacy IP-wired + Mooncake objects are never translated to Redis; a new typed MP Mooncake L2 + adapter remains future work. +- Current limits remain TP=1 for SGLang and exclude multi-node TP, distributed + executors, MLA, and MP-server restart/re-registration. ## Converged foundation: MP for both engines @@ -72,25 +34,24 @@ and each engine attaches through its own launch surface. | Engine | LMCache modes | Recommended (operator docs) | This design implements | |---|---|---|---| | **SGLang** | **MP only** — `LMCacheMPConnector` via `--lmcache-config-file`; no `lm://` client exists | MP (the only option) | **Yes — Phases 1–3** | -| **vLLM** | `lm://` (shipped: `LMCacheConnectorV1` + `LMCACHE_REMOTE_URL`) **and** MP (`LMCacheMPConnector` + `mp.host`/`mp.port`) | **MP** | vLLM MP is a **future migration** reusing this infra; the `lm://` adapter stays supported | +| **vLLM** | **MP only** — `LMCacheMPConnector` via `--kv-transfer-config` | MP | **Yes — typed PodLocal** | Policy this locks in: - **SGLang: MP-only.** No `lm://` client exists for SGLang, so the adapter supports MP exclusively (this design). -- **vLLM: both, MP recommended.** The existing `lm://` vLLM+LMCache adapter stays - supported (validated and shipped — operators on it are not broken). A future vLLM - MP adapter reuses the *same* worker + config/extra-config wire + shared-L2 store - this design builds; operator-facing docs **recommend MP** for vLLM once it lands, - matching upstream. +- **vLLM: MP-only.** The former IP adapter was removed in migration Phase 7. - **Shared, engine-agnostic infrastructure.** The node-local worker, the config-file / `kv_connector_extra_config` wire, the shared L2 (Redis / `--l2-adapter`), and the `/dev/shm` + fail-open handling are not SGLang-specific — SGLang is just the first consumer. Keeping them engine-neutral is what makes the vLLM migration a wiring change, not a rebuild. -Everything below specifies the SGLang implementation concretely; the engine-agnostic -pieces are flagged so the vLLM migration inherits them. +## Historical spike record + +Everything below describes the SGLang implementation as it existed during the +spike. It is retained as evidence and rationale, not as the current API or +support contract. ## Background: how SGLang+LMCache actually works @@ -492,8 +453,9 @@ data plane), different resolution because the data planes differ: silently falls back to slow pickle serialization. The shared `emptyDir` must be `medium: Memory` and sized ≥ the L1. - **L2 durability/HA** — a single managed Redis is a simple default, not an HA - store. A future typed remote-storage option will let operators - who need durability select an `s3` or `mooncake_store` `--l2-adapter` instead, - mirroring the LMCache-vs-Mooncake durability-is-a-backend-choice decision. + store. Future provider-specific work may add a real managed Redis Cluster; + separately, a typed `mooncake_store` L2 adapter can provide Mooncake without + restoring the deleted IP wire. Neither capability is represented by scaling + the standalone Redis Deployment. - **Bleeding edge** — SGLang's LMCache integration is new (early 2026); the working image/version tuple is pinned by the reference stack, not assumed stable. diff --git a/docs/observability/alerts.md b/docs/observability/alerts.md index 4469dc8c..35a2140b 100644 --- a/docs/observability/alerts.md +++ b/docs/observability/alerts.md @@ -32,8 +32,7 @@ There are two distribution shapes, same rule set, drift-gated by Required for the controller-side alerts (`ServerProbeFail` reads `inferencecache_backend_probe_result_total`, which the CacheBackend reconciler emits; the existing - `inferencecache_backend_server_restart_cascades_total` is also - controller-emitted). Without this, those rules load but never + `inferencecache_backend_probe_result_total` is controller-emitted). Without this, those rules load but never have a series to evaluate. 3. A second [`PodMonitor`](../../config/observability/lmcache-podmonitor.yaml) that discovers successfully injected PodLocal LMCache native sidecars @@ -81,8 +80,8 @@ There are two distribution shapes, same rule set, drift-gated by need `scrape_configs:` entries for all applicable targets — the `inference-cache-server` pod (server-side series: index, lookup, auth) AND the `inference-cache-controller-manager` pod (controller-side - series: per-stage probe-result counter, cache-server restart-cascade - counter) and each injected PodLocal LMCache sidecar (`:8080/metrics`). + per-stage probe-result counter) and each injected PodLocal LMCache sidecar + (`:8080/metrics`). Server-only scrape leaves the controller-side alerts (`ServerProbeFail` today) loaded but inert — they read `inferencecache_backend_probe_result_total` which is controller-emitted. diff --git a/docs/quickstart.md b/docs/quickstart.md index c45fa2a8..e6bd1e6b 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -256,7 +256,6 @@ install: - **The CR uses `spec.remoteStorage.ownership: External`.** Externally owned backends are wholly exempt from the probe gate — the controller never drives a round-trip against a cache it does not manage. -- **The CR is on an Unmanaged path** (unsupported runtime, deferred - `deploymentKind: StatefulSet`, or other reconcile branch that sheds - the managed workload). The probe gate is exempt from these paths +- **The CR is on an Unmanaged path** (for example, an unsupported runtime or + provider binding that bypassed admission). The probe gate is exempt from these paths and any prior `FunctionalProbeOK` condition is removed on transition. diff --git a/docs/reference-stack/README.md b/docs/reference-stack/README.md index 45d2537e..163728d2 100644 --- a/docs/reference-stack/README.md +++ b/docs/reference-stack/README.md @@ -22,7 +22,7 @@ LMCache server. Normal engine startup is the authoritative compatibility check. | [`manifests/deployment.yaml`](manifests/deployment.yaml) | vLLM + typed host-only LMCache MP. | | [`manifests/sglang-lmcache/`](manifests/sglang-lmcache/) | SGLang + typed LMCache MP + explicit external Redis. | | [`manifests/cpu-local/`](manifests/cpu-local/) | CPU-only engine/event check without LMCache. | -| [`scripts/`](scripts/) | Event subscriber, prefix-hit test, and compatibility canaries. | +| [`scripts/`](scripts/) | Event subscriber, prefix-hit test, and MP-only install smoke. | There is no Helm values reference in this phase. The repository has not validated an upstream chart API that can faithfully express the operator-owned @@ -65,11 +65,12 @@ kubectl apply -f manifests/deployment.yaml -f manifests/service.yaml kubectl -n cache-substrate rollout status deploy/vllm-lmcache-llama-8b --timeout=20m ``` -The vLLM reference is host-only: `status.endpoint` is intentionally empty. +The vLLM reference is host-only: `status.remoteStorage` is intentionally absent. For cross-Pod sharing, explicitly select Redis as shown by the SGLang reference or [`config/samples/cachebackend-lmcache.yaml`](../../config/samples/cachebackend-lmcache.yaml). -Legacy `LMCacheServer` and Mooncake providers are not automatically translated -because doing so would silently change L3 and sharing semantics. +The removed `LMCacheServer` and legacy IP-wired Mooncake provider shapes are not +automatically translated because doing so would silently change L3 and sharing +semantics. Mooncake remains future typed MP L2 work. ## Verify traffic and KV events @@ -114,13 +115,9 @@ python scripts/kv_events_subscriber.py --endpoint tcp://localhost:5557 --max 4 python scripts/test_kv_events.py ``` -## Legacy compatibility canaries - -`canary_c2_reconcile.sh` and `canary_c6_engine_wiring.sh` intentionally exercise -the legacy IP implementation retained until Phase 7. They are manual -compatibility tests, not current deployment references, and their workflows are -not scheduled. Current typed MP rendering and admission are covered by Go tests -and `default_install_smoke.sh`. +The `default_install_smoke.sh` gate installs the current MP-only schema, checks +typed Pod admission and managed Redis, and re-applies the bundle to cover the +in-place alpha upgrade path without starting a real engine. ## Teardown diff --git a/docs/reference-stack/VERSIONS.md b/docs/reference-stack/VERSIONS.md index 0961ba05..30bf370e 100644 --- a/docs/reference-stack/VERSIONS.md +++ b/docs/reference-stack/VERSIONS.md @@ -10,7 +10,7 @@ engine startup is the authoritative package/connector compatibility check. | Component | Reference value | Notes | |---|---|---| -| LMCache standalone server | `docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13` | Pinned by the typed `CacheBackend`; runs `lmcache server`. This exact reference digest is structurally tested, while the GPU evidence below used the separately recorded validation digest. | +| LMCache MP-server sidecar | `docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13` | Pinned by the typed `CacheBackend`; runs `lmcache server` as the PodLocal native sidecar. This exact reference digest is structurally tested, while the GPU evidence below used the separately recorded validation digest. | | vLLM engine | non-pullable all-zero placeholder | Replace with a digest-pinned image containing the LMCache MP connector/package. The repository deliberately supplies no default engine image. | | SGLang engine | non-pullable all-zero placeholder | Replace with a digest-pinned SGLang image containing a compatible LMCache client. The repository deliberately supplies no default engine image. | | Redis | `docker.io/library/redis:7.4-alpine` | Used only by the SGLang reference as an explicit external L3 choice. Digest-pin it for production. | @@ -18,7 +18,7 @@ engine startup is the authoritative package/connector compatibility check. | SGLang model | `meta-llama/Meta-Llama-3-8B-Instruct` | Gated; keep the served model, request model, and `observation.modelID` aligned. | | CPU-only engine | `vllm/vllm-openai-cpu:latest-{x86_64,arm64}` | Event/prefix-cache check only; no LMCache MP data plane. Mutable development tag, not a production pin. | -The standalone reference digest and the GPU-validation digest differ. Do not +The MP-server reference digest and the GPU-validation digest differ. Do not interpret structural manifest coverage as a claim that this exact engine/server tuple has completed the live GPU matrix. @@ -40,11 +40,12 @@ path with those exact test inputs; they are not universal image endorsements. Before production rollout: 1. Build or select the engine image in the inference-system release process. -2. Pin both engine and standalone-server images by digest. +2. Pin both engine and MP-server sidecar images by digest. 3. Create the typed `CacheBackend` and let the webhook inject MP wiring. 4. Treat engine startup/readiness as the compatibility verdict, then run a store, local-cache reset, and retrieve test on the target GPU/CUDA stack. -Do not map legacy `LMCacheServer` or Mooncake objects to Redis automatically. +Do not map removed `LMCacheServer` or legacy IP-wired Mooncake objects to Redis automatically. The operator must explicitly choose host-only MP or a supported L3 because the -choice changes cross-Pod sharing and persistence semantics. +choice changes cross-Pod sharing and persistence semantics. Mooncake support +returns only through a separately validated typed MP L2 adapter. diff --git a/docs/reference-stack/manifests/sglang-lmcache/README.md b/docs/reference-stack/manifests/sglang-lmcache/README.md index d101bd8e..4e443e2e 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/README.md +++ b/docs/reference-stack/manifests/sglang-lmcache/README.md @@ -17,7 +17,7 @@ same production renderer exercised by admission and controller tests. - One NVIDIA GPU; this TP=1 reference has no supported CPU fallback. - The inference-cache controller and mutating webhook installed first. - A digest-pinned SGLang engine image containing an LMCache client compatible - with the pinned standalone server. The manifest's all-zero digest is + with the pinned MP-server sidecar. The manifest's all-zero digest is deliberately non-pullable. - A Hugging Face token for the gated model. @@ -40,8 +40,9 @@ external Redis Service in this manifest ``` Redis is an explicit operator choice. Omitting `remoteStorage` produces a -host-only MP backend. A legacy LMCacheServer or Mooncake configuration is not -silently converted to Redis because that would change cross-Pod/L3 semantics. +host-only MP backend. A removed LMCacheServer or legacy IP-wired Mooncake +configuration is not silently converted to Redis because that would change +cross-Pod/L3 semantics. Mooncake requires a future typed MP L2 adapter. ## Deploy diff --git a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml index 24cfb452..e68819c5 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml +++ b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml @@ -5,7 +5,8 @@ # SGLang + typed LMCache multiprocess reference deployment. The CacheBackend # selects the engine pod and the operator injects the native lmcache-mp-server # sidecar, MP config, and SGLang connector arguments. Redis is an explicit shared -# tier choice; no legacy LMCacheServer or Mooncake topology is inferred as Redis. +# tier choice; no removed LMCacheServer or legacy IP-wired Mooncake topology is +# inferred as Redis. Mooncake requires a future typed MP L2 adapter. # # The engine image, model, GPU resources, and KV-event publisher remain inference- # system-owned. The engine image must contain a connector/package compatible with @@ -28,7 +29,7 @@ # IMAGE PINNING (see VERSIONS.md): the engine image is a deliberately non-pullable # placeholder because this repository does not own inference engine images. Replace # it with a digest-pinned compatible image. The CacheBackend pins the independently -# owned standalone server image. Digest-pin Redis for production as well. +# owned MP-server sidecar image. Digest-pin Redis for production as well. apiVersion: apps/v1 kind: Deployment metadata: diff --git a/docs/reference-stack/scripts/canary_c2_reconcile.sh b/docs/reference-stack/scripts/canary_c2_reconcile.sh deleted file mode 100755 index 620428dd..00000000 --- a/docs/reference-stack/scripts/canary_c2_reconcile.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# LEGACY IP COMPATIBILITY ONLY. Retained until Phase 7; do not use this script -# as a production deployment reference. It intentionally exercises -# remoteStorage.provider=LMCacheServer and the lm:// data plane. -# -# Canary for the C2 CacheBackend reconciler. Proves the controller stands up a -# healthy, serving backend from a CR on a GPU-free cluster (kind): -# -# kubectl apply CacheBackend(profile=cpu) --> controller --> Deployment + Service -# --> cache-server Deployment becomes Available --> Ready condition True, -# status.endpoint set -# -# Optionally drives prefix traffic and checks an engine prefix-cache hit — but -# this is opt-in (SKIP_TRAFFIC=0). The traffic block expects a vLLM HTTP surface -# on a port-forward target the script does NOT set up by default (the cache- -# server Service exposes only the LMCache TCP lm:// port). Operators wiring a -# vLLM engine alongside the canary need to also point the port-forward at the -# engine Service before flipping the toggle. Deleting the CR garbage-collects -# the children via owner refs. -# -# This exercises the reconciler end to end against real pods — the gap envtest -# can't cover. The managed standalone server uses CPU storage and does not need -# an inference engine or GPU for this controller lifecycle check. -# -# Manual canary (NOT a per-PR gate): needs Docker + kind + kubectl, pulls the -# standalone LMCache server image. See docs/reference-stack/VERSIONS.md. -# -# Usage: docs/reference-stack/scripts/canary_c2_reconcile.sh -# Tunables via env: CACHE_SERVER_IMAGE, MODEL, KIND_CLUSTER, NAMESPACE, -# READY_TIMEOUT, SKIP_TRAFFIC. -set -euo pipefail - -CACHE_SERVER_IMAGE="${CACHE_SERVER_IMAGE:-lmcache/standalone:v0.4.7}" -MODEL="${MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" -KIND_CLUSTER="${KIND_CLUSTER:-ic-c2-canary}" -NAMESPACE="${NAMESPACE:-c2-canary}" -CR_NAME="${CR_NAME:-canary}" -READY_TIMEOUT="${READY_TIMEOUT:-900}" # seconds for the CPU model to load + become Ready -# The traffic block port-forwards `svc/$CR_NAME` to a `:8000` vLLM HTTP -# surface and asserts a prefix-cache hit via the vllm:prefix_cache_hits_total -# metric — a holdover from the retired colocated-rendering profile that -# bundled vLLM into the cache-server pod. The modern split layout exposes -# only the LMCache server on `:65432` (TCP lm://), with no vLLM and no -# HTTP /metrics on the cache-server Service, so the traffic path cannot -# succeed unless the operator wires a separate engine Service AND repoints -# the port-forward below at it. Default the toggle OFF; operators who set up -# both pieces flip SKIP_TRAFFIC=0 to opt back in. -SKIP_TRAFFIC="${SKIP_TRAFFIC:-1}" - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -cd "$REPO_ROOT" - -KIND="${KIND:-$([ -x ./bin/kind ] && echo ./bin/kind || echo kind)}" -controller_pid="" -pf_pid="" -log() { echo "[c2-canary] $*"; } -fail() { - echo "[c2-canary] FAIL: $*" >&2 - exit 1 -} - -cleanup() { - [ -n "$pf_pid" ] && kill "$pf_pid" 2>/dev/null || true - [ -n "$controller_pid" ] && kill "$controller_pid" 2>/dev/null || true - "$KIND" delete cluster --name "$KIND_CLUSTER" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -# --- cluster ---------------------------------------------------------------- -log "creating kind cluster $KIND_CLUSTER" -"$KIND" create cluster --name "$KIND_CLUSTER" --wait 120s -KUBECONFIG_ARGS=(--context "kind-$KIND_CLUSTER") - -log "pulling LMCache server image and loading it into the node ($CACHE_SERVER_IMAGE)" -docker pull "$CACHE_SERVER_IMAGE" -"$KIND" load docker-image "$CACHE_SERVER_IMAGE" --name "$KIND_CLUSTER" - -# --- controller ------------------------------------------------------------- -log "installing CRD" -kubectl "${KUBECONFIG_ARGS[@]}" apply -f config/crd/bases/inferencecache.io_cachebackends.yaml - -log "building + starting the controller" -go build -o bin/controller ./cmd/controller - -# The controller registers its admission webhooks unconditionally, so the -# manager starts an in-process webhook server that reads its TLS serving -# cert from this directory at startup — mgr.Start() returns an error and the -# whole manager exits if tls.crt/tls.key are absent, before the reconciler -# ever runs. This canary installs no WebhookConfiguration (only the CRD), so -# the apiserver never calls the webhook; the cert only has to exist for the -# server to bind. Mint a throwaway self-signed pair — nothing verifies it. -# The cert dir must match controller-runtime's default webhook CertDir, -# which is os.TempDir()/k8s-webhook-server/serving-certs — i.e. honour -# TMPDIR (unset on the CI runner, so this resolves to /tmp there). -webhook_cert_dir="${TMPDIR:-/tmp}/k8s-webhook-server/serving-certs" -mkdir -p "$webhook_cert_dir" -openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/CN=c2-canary-webhook" \ - -keyout "$webhook_cert_dir/tls.key" -out "$webhook_cert_dir/tls.crt" >/dev/null 2>&1 - -./bin/controller --leader-elect=false >/tmp/c2-canary-controller.log 2>&1 & -controller_pid=$! - -kubectl "${KUBECONFIG_ARGS[@]}" create namespace "$NAMESPACE" - -# --- apply the CacheBackend -------------------------------------------------- -log "applying CacheBackend $NAMESPACE/$CR_NAME (image=$CACHE_SERVER_IMAGE)" -kubectl "${KUBECONFIG_ARGS[@]}" apply -f - </dev/null || true)" - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" get pods -o wide || true - kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" describe deployment "$CR_NAME" || true - fail "backend did not become Ready within ${READY_TIMEOUT}s (last Ready status='$ready')" - fi - sleep 5 -done -log "Ready=True" - -endpoint="$(kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" get cachebackend "$CR_NAME" -o jsonpath='{.status.endpoint}')" -[ -n "$endpoint" ] || fail "status.endpoint was not published" -log "status.endpoint=$endpoint" - -avail="$(kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" get deployment "$CR_NAME" -o jsonpath='{.status.availableReplicas}')" -[ "${avail:-0}" -ge 1 ] || fail "deployment has no available replicas" - -# --- optional: drive prefix traffic + check a cache hit --------------------- -if [ "$SKIP_TRAFFIC" != "1" ]; then - log "port-forwarding the Service to drive prefix traffic" - kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" port-forward "svc/$CR_NAME" 18000:8000 >/tmp/c2-canary-pf.log 2>&1 & - pf_pid=$! - for _ in $(seq 1 30); do - curl -sf -o /dev/null "http://localhost:18000/health" && break - sleep 1 - done - hits() { curl -s "http://localhost:18000/metrics" | awk '/^vllm:prefix_cache_hits_total/{s+=$2} END{print s+0}'; } - PREFIX="$(python3 -c 'print(("You are a meticulous canary assistant. Follow the rules precisely. " * 200).strip())')" - fire() { - curl -s -o /dev/null -w '%{http_code}' "http://localhost:18000/v1/chat/completions" \ - -H 'Content-Type: application/json' \ - -d "$(python3 -c 'import json,sys;print(json.dumps({"model":sys.argv[3],"max_tokens":8,"temperature":0,"messages":[{"role":"system","content":sys.argv[1]},{"role":"user","content":sys.argv[2]}]}))' "$PREFIX" "$1" "$MODEL")" - } - h0=$(hits) - log "request 1 (cold prefix): HTTP $(fire 'summarize in one word')" - log "request 2 (same prefix): HTTP $(fire 'summarize in two words')" - h1=$(hits) - log "prefix_cache_hits: $h0 -> $h1" - [ "$h1" -gt "$h0" ] || fail "no engine prefix-cache hit (hits did not increase)" -fi - -# --- delete the CR -> owner-ref GC ------------------------------------------ -log "deleting the CR; expecting owner-ref GC of the Deployment + Service" -kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" delete cachebackend "$CR_NAME" --wait=true -gc_deadline=$(($(date +%s) + 60)) -until [ "$(kubectl "${KUBECONFIG_ARGS[@]}" -n "$NAMESPACE" get deploy,svc -o name 2>/dev/null | wc -l | tr -d ' ')" = "0" ]; do - [ "$(date +%s)" -lt "$gc_deadline" ] || fail "children were not garbage-collected after CR deletion" - sleep 2 -done - -log "PASS — reconciler stood up a healthy backend, published its endpoint, and cleaned up on delete" diff --git a/docs/reference-stack/scripts/canary_c6_engine_wiring.sh b/docs/reference-stack/scripts/canary_c6_engine_wiring.sh deleted file mode 100755 index 73a5d7ee..00000000 --- a/docs/reference-stack/scripts/canary_c6_engine_wiring.sh +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# LEGACY IP COMPATIBILITY ONLY. Retained until Phase 7; do not use this script -# as a production deployment reference. It intentionally exercises -# LMCacheConnectorV1, LMCACHE_REMOTE_URL, and LMCacheServer. -# -# CPU canary for the C6 mutating Pod webhook + cross-pod cache reuse. -# -# Proves the engine-wiring webhook injects the LMCache connector env onto -# user-supplied vLLM pods at admission time, and (when not skipped) that two -# such pods share KV state via a single managed lmcache-server: -# -# apply CacheBackend(type=LMCache, engineSelector={app: vllm-engine}) -# -> reconciler stands up lmcache-server (Ready) -# -> apply Pod A (label app=vllm-engine) -- webhook injects LMCACHE_* -# -> apply Pod B (label app=vllm-engine) -- webhook injects LMCACHE_* -# -# Then (SKIP_TRAFFIC != 1) sends the same long-prefix prompt to Pod A then -# Pod B, and confirms Pod B reports a vllm:prefix_cache_hits increment that -# Pod A did not produce from its own cold start -- the shared lmcache-server -# made the prefix available to Pod B without re-prefill. -# -# Heavy: two CPU vLLM pods + the lmcache-server + cert-manager + the -# controller image all run on one kind node. The vLLM CPU runtime baseline -# is ~5 GiB per pod, so the Docker VM needs ~12 GiB RAM (see -# docs/reference-stack/VERSIONS.md for the documented memory floor). -# Pulls the multi-GB vLLM image. This is NOT a per-PR gate; it runs on a -# manual dispatch only. -# -# Usage: docs/reference-stack/scripts/canary_c6_engine_wiring.sh -# Tunables: IMAGE, MODEL, KIND_CLUSTER, NAMESPACE, READY_TIMEOUT, -# SKIP_TRAFFIC, CERT_MANAGER_VERSION. - -set -euo pipefail - -arch="$(uname -m)" -case "$arch" in - arm64 | aarch64) IMAGE_TAG="${IMAGE_TAG:-latest-arm64}" ;; - *) IMAGE_TAG="${IMAGE_TAG:-latest-x86_64}" ;; -esac -IMAGE="${IMAGE:-vllm/vllm-openai-cpu:$IMAGE_TAG}" -MODEL="${MODEL:-Qwen/Qwen2.5-0.5B-Instruct}" -KIND_CLUSTER="${KIND_CLUSTER:-ic-c6-canary}" -NAMESPACE="${NAMESPACE:-c6-canary}" -CR_NAME="${CR_NAME:-canary-lmcache}" -READY_TIMEOUT="${READY_TIMEOUT:-900}" -SKIP_TRAFFIC="${SKIP_TRAFFIC:-0}" -CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.1}" -CONTROLLER_IMG="${CONTROLLER_IMG:-localhost/inference-cache-controller:canary}" -SERVER_IMG="${SERVER_IMG:-localhost/inference-cache-server:canary}" - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -cd "$REPO_ROOT" - -KIND="${KIND:-$([ -x ./bin/kind ] && echo ./bin/kind || echo kind)}" -pf_pids=() -log() { echo "[c6-canary] $*"; } -fail() { - echo "[c6-canary] FAIL: $*" >&2 - exit 1 -} - -cleanup() { - for pid in "${pf_pids[@]}"; do - kill "$pid" 2>/dev/null || true - done - "$KIND" delete cluster --name "$KIND_CLUSTER" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -# --- cluster ---------------------------------------------------------------- -log "creating kind cluster $KIND_CLUSTER" -"$KIND" create cluster --name "$KIND_CLUSTER" --wait 120s -KCTX=(--context "kind-$KIND_CLUSTER") - -log "installing cert-manager $CERT_MANAGER_VERSION (webhook serving cert)" -kubectl "${KCTX[@]}" apply -f \ - "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" -kubectl "${KCTX[@]}" -n cert-manager wait --for=condition=Available deployment --all --timeout=180s - -# --- controller + server images + install ---------------------------------- -log "building controller image $CONTROLLER_IMG" -docker build -f dockerfiles/Dockerfile --target controller -t "$CONTROLLER_IMG" . -"$KIND" load docker-image "$CONTROLLER_IMG" --name "$KIND_CLUSTER" - -log "building server image $SERVER_IMG" -docker build -f dockerfiles/Dockerfile --target server -t "$SERVER_IMG" . -"$KIND" load docker-image "$SERVER_IMG" --name "$KIND_CLUSTER" - -log "rendering + applying config/default (controller + server + webhook + cert-manager wiring)" -# Point both the controller and server Deployments at our canary images. -# The default overlay now ships an inference-cache-server Deployment too; -# without rewriting its image the canary would resolve to the published -# :dev tag, not the image built from this commit. Prefer `kustomize edit` -# when available; otherwise fall back to a sed that scopes the rewrite to -# each `- name: ...` block (both blocks share `newTag: dev`, so an -# unscoped substitute would collapse them onto the same value). -tmpdir="$(mktemp -d)" -trap "rm -rf $tmpdir; cleanup" EXIT -cp -r config "$tmpdir/config" -( - cd "$tmpdir/config/default" - if command -v kustomize >/dev/null 2>&1; then - kustomize edit set image controller="$CONTROLLER_IMG" server="$SERVER_IMG" - else - # Split on the LAST `:` so refs that include a registry port - # (e.g. localhost:5001/inference-cache-server:canary) keep their - # full registry/repo path. `${X%:*}` strips the shortest suffix - # from the last `:`, leaving everything before the tag. - sed -i.bak \ - -e "/^- name: controller$/,/^- name: server$/ { - s|^ newName: .*| newName: ${CONTROLLER_IMG%:*}| - s|^ newTag: .*| newTag: ${CONTROLLER_IMG##*:}| - }" \ - -e "/^- name: server$/,\$ { - s|^ newName: .*| newName: ${SERVER_IMG%:*}| - s|^ newTag: .*| newTag: ${SERVER_IMG##*:}| - }" \ - kustomization.yaml - fi -) -kubectl "${KCTX[@]}" apply -k "$tmpdir/config/default" -kubectl "${KCTX[@]}" -n inference-cache-system wait --for=condition=Available deployment --all --timeout=180s - -# --- pre-existing namespace ------------------------------------------------ -kubectl "${KCTX[@]}" create namespace "$NAMESPACE" - -# --- apply the CacheBackend ------------------------------------------------ -log "applying CacheBackend $NAMESPACE/$CR_NAME (engineSelector matches app=vllm-engine)" -kubectl "${KCTX[@]}" apply -f - </dev/null)" = "True" ]; do - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl "${KCTX[@]}" -n "$NAMESPACE" get pods -o wide || true - kubectl "${KCTX[@]}" -n "$NAMESPACE" describe cachebackend "$CR_NAME" || true - fail "lmcache-server did not become Ready within ${READY_TIMEOUT}s" - fi - sleep 5 -done -endpoint="$(kubectl "${KCTX[@]}" -n "$NAMESPACE" get cachebackend "$CR_NAME" -o jsonpath='{.status.endpoint}')" -log "CacheBackend Ready; endpoint=$endpoint" - -# --- two engine pods, labels matching the EngineSelector ------------------- -log "loading vLLM CPU image into the kind node ($IMAGE)" -docker pull "$IMAGE" -"$KIND" load docker-image "$IMAGE" --name "$KIND_CLUSTER" - -apply_engine_pod() { - local name="$1" - kubectl "${KCTX[@]}" apply -f - <&2 || true - fail "webhook did not inject LMCACHE_REMOTE_URL onto $name" - fi - log "verified $name carries LMCACHE_REMOTE_URL=$remote_url" -} -verify_webhook_inject "engine-a" -verify_webhook_inject "engine-b" - -if [ "$SKIP_TRAFFIC" = "1" ]; then - log "SKIP_TRAFFIC=1 -> skipping traffic + cache-hit assertion" - log "PASS - webhook injected wiring on both engine pods (traffic step skipped)" - exit 0 -fi - -# Wait for both engine pods to be Ready (heavy CPU model load). -for name in engine-a engine-b; do - log "waiting up to ${READY_TIMEOUT}s for $name to become Ready" - kubectl "${KCTX[@]}" -n "$NAMESPACE" wait --for=condition=Ready --timeout="${READY_TIMEOUT}s" "pod/$name" \ - || fail "$name did not become Ready" -done - -# Port-forward each engine's HTTP port to a distinct local port so we can -# hit them independently and read their /metrics. -forward_pod() { - local name="$1" local_port="$2" - kubectl "${KCTX[@]}" -n "$NAMESPACE" port-forward "pod/$name" "$local_port:8000" \ - >"/tmp/c6-canary-pf-$name.log" 2>&1 & - pf_pids+=($!) - for _ in $(seq 1 30); do - curl -sf -o /dev/null "http://localhost:$local_port/health" && return 0 - sleep 1 - done - fail "engine $name port-forward never became healthy" -} -forward_pod engine-a 18001 -forward_pod engine-b 18002 - -hits() { - local port="$1" - curl -s "http://localhost:$port/metrics" | awk '/^vllm:prefix_cache_hits_total/{s+=$2} END{print s+0}' -} - -PREFIX="$(python3 -c 'print(("You are a meticulous canary assistant. Follow the rules precisely. " * 200).strip())')" -fire() { - local port="$1" q="$2" - curl -s -o /dev/null -w '%{http_code}' "http://localhost:$port/v1/chat/completions" \ - -H 'Content-Type: application/json' \ - -d "$(python3 -c 'import json,sys;print(json.dumps({"model":sys.argv[3],"max_tokens":8,"temperature":0,"messages":[{"role":"system","content":sys.argv[1]},{"role":"user","content":sys.argv[2]}]}))' "$PREFIX" "$q" "$MODEL")" -} - -a_h0=$(hits 18001); b_h0=$(hits 18002) -log "cold counters: engine-a vllm:prefix_cache_hits=$a_h0 ; engine-b=$b_h0" - -log "request 1 to engine-a (cold; should produce no prefix-cache hit on A)" -log " HTTP $(fire 18001 'summarize in one word')" -a_h1=$(hits 18001) -log "engine-a hits delta after self-cold prompt: $((a_h1 - a_h0))" - -log "request 2 to engine-b (same prefix; SHOULD hit via the shared lmcache-server)" -log " HTTP $(fire 18002 'summarize in two words')" -b_h1=$(hits 18002) -log "engine-b hits delta after cross-pod prompt: $((b_h1 - b_h0))" - -# Pod A's first request was cold so it should have produced no hit on A. -# Pod B's first request, with the same prefix, must hit via the shared -# lmcache-server -- otherwise the webhook did not actually wire the engines -# into the same backend (which is the contract this canary asserts). -if [ "$((b_h1 - b_h0))" -le 0 ]; then - log "engine-a metrics tail:"; curl -s http://localhost:18001/metrics | grep -E '^vllm:prefix_cache_' || true - log "engine-b metrics tail:"; curl -s http://localhost:18002/metrics | grep -E '^vllm:prefix_cache_' || true - fail "engine-b reported no prefix-cache hit increment for a prompt prefix that engine-a had populated via the shared lmcache-server" -fi - -log "PASS - webhook auto-wired both engines and the shared lmcache-server delivered a cross-pod prefix-cache hit" diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index cbafdbf9..ca91b16c 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -4,4360 +4,326 @@ # # SPDX-License-Identifier: Apache-2.0 -# Phase-5 classification: typed PodLocal MP checks in this script are current. -# Every LMCacheServer, LMCacheConnectorV1/LMCACHE_REMOTE_URL, lm://, or Mooncake -# case is an intentional legacy-IP compatibility assertion retained only until -# Phase 7; none is a production fixture or recommended deployment path. - -# Per-PR install smoke for `kubectl apply -k config/default`. -# -# Builds controller + server images at a deterministic tag, loads them into a -# kind cluster, installs cert-manager (same pinned version the C6 engine-wiring -# canary uses), renders config/default with the SHA-tagged images, applies it, -# and asserts the install actually came up: -# -# 1. inference-cache-controller-manager + inference-cache-server reach -# condition=Available within 120s. -# 2. The CacheIndex poller is writing status: `cacheindex/cluster-default` -# has a non-empty `.status.observedServer` within ~60s (one or two poll -# cycles past the 30s default refresh). -# 3. The server's operator HTTP surface is wired on the installed Service: -# `/readyz` returns 200, `/metrics` exposes `inferencecache_server_up 1`, -# and `kubectl get ci` renders the CacheIndex Prefixes/Changed printer -# columns. -# 4. The CachePolicy PUSH path works: an applied `CachePolicy` renders its -# operator-facing printer columns, the controller pushes it to the -# server's `/policy` endpoint, and `LookupRoute` observes ALL THREE -# policy enforcement paths without engine pods or inference traffic: -# the pushed `minimumPrefixTokens` request-side gate, the pushed -# `minimumMatchedTokens` per-replica result-side floor, and the -# pushed `routingFloorScore` whole-response score floor. Three -# orthogonal lookups exercise the first two (above both, below the -# request-side gate, sub-floor realized match); a follow-up patch + -# re-lookup pair exercises the routingFloorScore propagation and -# replace-on-write semantics. The installed validating webhook also -# rejects a SECOND CachePolicy in the namespace (one-per-namespace), -# proving the bundle's webhook Service + cert-manager CA-injection -# path — not just envtest handler logic. -# 5. The per-CacheTenant status projection works: an applied `CacheTenant` -# gets `.status.indexEntries=0` (observed-zero — no engine traffic in the -# smoke) and a `Ready=True` condition written by the same poller. The -# installed validating webhook also rejects a SECOND CacheTenant reusing -# an existing tenantID in the namespace (tenantID-uniqueness). -# 6. PromptTemplate + PDTopology are schema-only in the default install -# today: the manager registers their CRDs/RBAC but no status-writing -# reconciler. The smoke applies committed samples and asserts -# `kubectl get pt` / `kubectl get pdt` render their operator-facing -# printer columns. -# 7. The gRPC surface is reachable and PLAINTEXT by default: config/default -# serves :9090 plaintext (TLS is opt-in — phase 13), so a plaintext client -# lists services and a `LookupRoute` for an unknown model returns the -# fail-open default (`reason_code: NO_HINT`). -# 8. The CacheBackend ↔ engine-pod binding surfaces operators rely on -# actually wire up end-to-end: applying config/samples/cachebackend- -# with-engine.yaml drives status.matchedEnginePods=1, stamps the -# injected-by annotation on the engine pod, and surfaces the -# InjectedByCacheBackend Event (with the persisted pod UID — the -# regression that hides events from `kubectl describe pod`). Then -# the cache-server restart cascade: force-deleting the -# cache-server pod flips status.observedServerInstance to the -# replacement's server-instance identifier and patches the cascade-restart-trigger -# annotation onto the engine Deployment's pod template (the -# mechanism that drives the rolling restart). Finally scaling the -# engine to 0 drives status.matchedEnginePods=0 via the -# reconciler's self-RequeueAfter cadence (no CR or owned-workload -# event needed) within ~30s, the bound on stale-Matched the -# cadence guarantees. -# 8a. The managed LMCache image is operator-configurable: the installed -# controller carries `--lmcache-server-image`, the smoke rewrites it to a -# locally built stand-in, and a CacheBackend with no CR-level image renders -# that configured image into its owned Deployment. -# 8b. Legacy-IP compatibility: an inline fixture leaves -# remoteStorage.lmCacheServer.resources unset, while the provider renderer -# gives the cache-server container a 4Gi request / 8Gi limit. The smoke -# asserts the CR remains unchanged and the rendered pod is still bounded -# against the T2-write OOM failure mode. -# 8c. The canonical cache hierarchy keeps engine wiring and provider -# lifecycle independent: the committed SGLang host-only sample creates no -# Deployment/Service/HPA and publishes no endpoint, while the committed -# SGLang+Managed-Redis sample explicitly creates a redis-l2 Deployment + -# Service and publishes its RESP endpoint. No engine traffic is required. -# 8d. Typed SGLang PodLocal admission: a matching, connector-declared SGLang -# Pod is actually persisted through the installed mutating webhook while -# pinned to an impossible node selector. The smoke reads the persisted Pod -# back and asserts the common lmcache-mp-server native sidecar, probes, -# resources, shared mounts, engine flags/env, and injection identity. -# 8e. Typed vLLM PodLocal admission: a matching, connector-declared vLLM Pod -# is persisted through the same live webhook and carries the dedicated -# LMCacheMPConnector module path, loopback MP endpoint, deterministic hash -# seed, hybrid-manager guard, and no legacy lm:// environment. -# 9. Legacy-IP External compatibility: an inline fixture drives the CacheBackend -# mutating webhook default (spec.replicas=1), renders NO -# Deployment/Service in its namespace, status.endpoint mirrors -# spec.remoteStorage.endpoint, observedGeneration is set, the CR goes -# Ready=True/ExternalEndpointAccepted, and -# `kubectl get cb` renders the CacheBackend printer columns. A matching -# engine pod is admitted with -# `LMCACHE_REMOTE_URL=lm://` -# injected by the pod-mutating webhook. Also exercises admission -# validation rules (External with no endpoint, External with bad -# endpoint shape, and non-External + endpoint are rejected at write time), -# plus the scale-to-zero guard: a CacheBackend with spec.replicas=0 + -# spec.autoscaling enabled + nil spec.autoscaling.minReplicas is rejected -# at admission and NOT persisted (the operator-facing surface added by the -# defaulter-sweep; without the rule a "scale to zero" intent would silently -# become "scale 1-N" via the reconciler's HPA fallback). -# 9b. The Events-only CacheBackend mode (spec.integration.mode=EventsOnly) -# end-to-end: applying an events-only LMCache CacheBackend renders NO owned -# Deployment/Service, keeps status.endpoint empty, latches no -# firstKVEventObservedAt (no subscriber image wired → no KV events), and -# parks the CR at Ready=False/AwaitingFirstKVEvent via the same KV-event -# gate as a managed backend. The managed-only conditions (FunctionalProbeOK -# / EngineKernelsHealthy / T2Degraded / EngineCompatibility) are absent. -# Also exercises the validating webhook's EventsOnly+External rejection. -# 9c. Native SGLang HiCache end-to-end: applying the committed HiCache sample -# is observed by the controller without rendering a Deployment, Service, or -# HPA; status.endpoint stays empty and no synthetic Ready condition is -# published. A matching engine Pod sent through real server-side dry-run -# admission receives the complete native --hicache-* argument contract, -# proving the endpoint-free Pod webhook path without starting SGLang. -# 10. The /snapshot endpoint rejects unauthenticated callers AT THE NETWORK -# LAYER: a side curl pod outside the controller's SA identity (and outside -# the NetworkPolicy allowlist) has its connection to :8081 DROPPED by the -# server NetworkPolicy, so curl times out (curl_failed:28). The cluster -# runs a NetworkPolicy-enforcing CNI (Calico), so a bare HTTP 401 — the L7 -# auth-middleware fallback — now FAILS the assertion: it would mean the -# NetworkPolicy was deleted/broken and only the auth middleware is left -# standing, the exact regression this gate exists to catch. -# 11. The /policy endpoint rejects unauthenticated callers at the network -# layer: same side-pod shape against the write-side endpoint. This is the -# more dangerous of the two — /policy is replace-on-write, so a successful -# unauthenticated POST would override every namespace's CachePolicy state -# cluster-wide. The probe POSTs a valid snapshot body so the rejection -# cannot be misattributed to a 400; the only valid outcome is the -# NetworkPolicy drop (curl_failed:28) — a bare 401 now FAILS. -# 11b. The /probe endpoint rejects unauthenticated callers at the network -# layer: same side-pod shape against the functional-self-test endpoint. -# /probe shares the controller ServiceAccount identity with /snapshot and -# /policy, so a regression that wired /probe outside that profile would let -# any pod that can reach :8081 drive a synthetic round-trip AND, since the -# CacheBackend reconciler now consumes the result to publish -# FunctionalProbeOK and downgrade Ready, observe or trigger forged Ready -# transitions on every managed backend. Sends a valid ProbeRequest body so -# the rejection cannot be misattributed to a 400; the only valid outcome is -# the NetworkPolicy drop (curl_failed:28) — a bare 401 now FAILS. -# 12. The audience binding holds on /snapshot, /policy, AND /probe: a probe -# pod with the controller's SA + labels reads three tokens -# (controller-audience projected, policy-audience projected, and the -# default-audience apiserver automount). It asserts the controller token -# admits on /snapshot + /probe, the policy token admits on /policy, and -# the default-audience token of the SAME SA is rejected everywhere; it -# also asserts the controller token cannot push /policy. Catches a -# regression in the SERVER's audience-enforcement half of the contract — -# `--controller-audience` / `--policy-audience` flag drift, the -# middleware forgetting to populate `TokenReviewSpec.Audiences`, or the -# apiserver mis-enforcing audience. Does NOT catch drift in the -# controller's production projected-volume manifest (the probe -# deliberately uses its own inline volume specs so it still runs when -# that manifest is broken); that drift is caught by item 2 above — -# observedServer populates only when the REAL controller's poller -# successfully scrapes /snapshot, and the CachePolicy adoption assertion -# passes only when the REAL controller's policy pusher reaches /policy. -# 12b. The authenticated /probe handler returns the expected default -# posture on a clean install: a controller-SA-authenticated POST -# gets HTTP 200 AND the parsed JSON body asserts ingest=ok, -# routing=ok, t2=skipped (no T2Prober is wired into the server -# today, so Stage C always reports skipped). A regression where -# the handler returns 200 with a per-stage "failed" would -# otherwise slip past the audience-binding phase above (which only -# checks HTTP status). -# 12c. The CacheTenant admission webhook rejects a CR claiming the -# server-reserved probe tenantID (inferencecache.io/probe). Pairs -# with the existing duplicate-tenantID assertion to pin BOTH -# CacheTenant validation rules end-to-end against the real -# installed webhook. -# 13. The opt-in gRPC TLS path works: applying config/overlays/server-tls -# (config/default + the config/server/tls component) rolls the server with -# --tls-cert-file/--tls-key-file + the cert-manager Secret. After rollout, -# a plaintext client is rejected and the cert-manager-issued chain + -# Service-FQDN SAN VERIFY against the CA published in the serving Secret -# (`grpcurl -cacert` with -authority ; a wrong authority is -# rejected) — proving server authentication, not just encryption, for the -# overlay operators actually enable. Finally re-runs the SAME -# LookupRoute(unknown model) the plaintext phase (7) ran and asserts the -# identical fail-open NO_HINT, proving the existing call pattern is -# unchanged over TLS (pure transport wrapper, no contract/behavior change). -# 14. The LMCache kernel-check injection shape is correct end-to-end: a -# GPU-requesting engine pod (labeled app=kc-inject-engine, bound to a -# dedicated LMCache CacheBackend) is admitted and carries a -# lmcache-kernel-check init container whose image EQUALS the engine -# container's image (the adapter reuses it so no extra image pull -# occurs). Exercises the mutating pod webhook's auto mode (inject -# iff GPU requested) end-to-end on the real installed bundle. -# 15. The report-only FAIL condition path works fail-open: a dedicated -# LMCache CacheBackend (kc-cond) is annotated report-only, and a -# matching engine pod using python:3.11-slim runs the kernel-check -# init container, which exits 0 (fail-open) but writes "FAIL: lmcache -# not importable" to /dev/termination-log. The main container starts -# normally (pod Ready), proving report-only did not block the engine. -# The C2 reconciler reads the termination message and publishes -# EngineKernelsHealthy=False / reason=KernelLoadFailed on the -# CacheBackend status. The validating webhook also rejects an invalid -# lmcache-kernel-check annotation value (a typo would otherwise silently -# relax strict enforcement to report-only). -# 16. Every sample manifest under config/samples/ applies cleanly against -# the live install: a server-side dry-run apply of each *.yaml/*.yml -# exercises CRD structural validation + the validating admission webhook -# on the real cluster. Complements `make verify-samples` (which runs the -# same assertion at envtest level) by catching admission-wiring failures -# envtest masks — the webhook being unreachable/mis-wired on a real -# cluster (cert-manager caBundle injection) and the CRDs as actually -# installed by config/default. Mirrors verify-samples' sample set and -# honors its `# verify-samples: skip` opt-out so the two gates stay in -# lockstep. Admission-level only — does NOT create CRs, write status, or -# hit /policy+/snapshot (no NetworkPolicy/RBAC coverage; the per-CRD -# phases above cover those). No engine pods, no traffic. -# 17. The operator `inferencecache doctor` CLI runs end-to-end against the live -# install: build the binary, apply a CacheBackend, run the config-only -# checks, and assert it emits the documented JSON envelope, surfaces a -# CacheBackend (CB0xx) finding, and exits with a code matching the reported -# summary.exitCode (the CI-gating contract). -# 18. The managed Mooncake backend reconciles end-to-end: a busybox -# `mooncake_master` stand-in (accepts TCP on the RPC port so the rendered -# readiness probe passes — the real kvcacheai/mooncake image is NOT pulled) -# lets `CacheBackend{type: LMCache, remoteStorage.provider: Mooncake}` -# reach an Available Deployment, with -# `status.endpoint=:50051` and the Service's first port = the RPC port. -# Proves the real installed controller selects the vLLM/LMCache adapter with -# a Mooncake binding and renders the mooncake_master provider workload; the real -# engine-over-mooncakestore:// path stays for the Mooncake reference stack. -# -# Distinct from the C2/C6 canaries: those exercise real engine pods + cross-pod -# cache reuse (multi-GB image, ~10+ GiB RAM, schedule-only). This smoke stops -# at "the default install bundle wires together; gRPC fail-open works; the -# CacheBackend ↔ engine-pod binding surfaces operators rely on actually -# wire up end-to-end" -- light enough to run on every PR. The paired-sample -# phase swaps the engine container's image to busybox before pod CREATE and -# uses a tiny locally built lmcache_server stand-in for the managed cache -# server, so the smoke does not pay multi-GB pulls or depend on mutable -# upstream image availability; the signals it asserts materialize from pod -# CREATE and the controller-managed Deployment readiness surface. -# -# Designed to catch the class of install regression that surfaced when the -# default overlay was missing a Namespace resource: `kubectl apply -k` silently -# fails namespace-scoped creates on a clean cluster, and the heavier canaries' -# `wait --for=condition=Available` mask it. -# -# Prereqs (fresh kind cluster + this repo, nothing else): -# - docker (for `make image-build` and `kind load docker-image`) -# - kind (./bin/kind picked up if present, else `kind` on PATH) -# - kubectl -# - curl (probes the installed HTTP surface) -# - grpcurl (probes the gRPC surface) -# - kustomize (optional; sed fallback handles the image rewrite if absent) -# -# Usage: docs/reference-stack/scripts/default_install_smoke.sh -# Tunables: TAG, KIND_CLUSTER, NAMESPACE, CERT_MANAGER_VERSION, CALICO_VERSION, -# READY_TIMEOUT, CACHEINDEX_TIMEOUT, POLICY_PUSH_TIMEOUT, HTTP_LOCAL_PORT, -# GRPC_LOCAL_PORT, LOG_DIR, POLICY_SMOKE_NS, SAMPLE_NS, -# PROMPT_TOPOLOGY_SMOKE_NS, SAMPLE_ENDPOINT_TIMEOUT, -# SAMPLE_MATCH_TIMEOUT, SAMPLE_DRIFT_TIMEOUT, -# SAMPLE_CASCADE_TIMEOUT, SAMPLE_ENGINE_IMAGE, -# SAMPLE_CACHE_SERVER_IMAGE, CANONICAL_BACKEND_TIMEOUT, -# CANONICAL_SMOKE_NS, EXTERNAL_BACKEND_TIMEOUT, -# EXTERNAL_INJECT_TIMEOUT, EVENTSONLY_BACKEND_TIMEOUT, -# EVENTSONLY_SMOKE_NS, EVENTSONLY_SMOKE_CB_NAME, SAMPLE_APPLY_NS, -# HICACHE_SMOKE_TIMEOUT, HICACHE_SMOKE_NS, HICACHE_SMOKE_CB_NAME, -# KERNEL_CHECK_SMOKE_NS, KERNEL_CHECK_POD_TIMEOUT, -# KERNEL_CHECK_COND_TIMEOUT, MOONCAKE_SMOKE_NS, MOONCAKE_MASTER_IMAGE. +# Per-PR smoke for the current MP-only CacheBackend contract. It installs +# config/default into a fresh kind cluster and verifies the served schema, +# managed Redis lifecycle, and real Pod admission mutation. No GPU or inference +# engine is required: engine startup remains the authoritative package/ +# connector compatibility check and is covered by the GPU validation matrix. set -euo pipefail TAG="${TAG:-${GITHUB_SHA:-$(git rev-parse HEAD)}}" +REGISTRY="${REGISTRY:-ghcr.io/cachebox-project}" KIND_CLUSTER="${KIND_CLUSTER:-ic-install-smoke}" -NAMESPACE="${NAMESPACE:-inference-cache-system}" +SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-inference-cache-system}" +SMOKE_NAMESPACE="${SMOKE_NAMESPACE:-ic-mp-smoke}" CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.1}" -# NetworkPolicy-enforcing CNI installed in place of kind's default kindnet (which -# does NOT enforce NetworkPolicy). Required so the server NetworkPolicy actually -# drops unauthenticated traffic to :8081 and the /snapshot + /policy + /probe -# assertions below can require the L3/L4 drop. Calico's default IPv4 -# pool is 192.168.0.0/16 — the kind podSubnet is set to match (see cluster block). -CALICO_VERSION="${CALICO_VERSION:-v3.28.2}" -READY_TIMEOUT="${READY_TIMEOUT:-120s}" -CACHEINDEX_TIMEOUT="${CACHEINDEX_TIMEOUT:-90}" # seconds; ~3x the 30s refresh, absorbs leader-election + first-tick jitter -POLICY_PUSH_TIMEOUT="${POLICY_PUSH_TIMEOUT:-90}" # seconds; watch-triggered push + one 30s periodic repair tick -HTTP_LOCAL_PORT="${HTTP_LOCAL_PORT:-18080}" +READY_TIMEOUT="${READY_TIMEOUT:-180s}" +CACHEINDEX_TIMEOUT="${CACHEINDEX_TIMEOUT:-90}" GRPC_LOCAL_PORT="${GRPC_LOCAL_PORT:-19090}" +HTTP_LOCAL_PORT="${HTTP_LOCAL_PORT:-18080}" +KEEP_CLUSTER="${KEEP_CLUSTER:-0}" LOG_DIR="${LOG_DIR:-/tmp/install-smoke-logs}" -# External-backend gate timeouts. The reconciler patches status on the next -# reconcile (sub-second on a fresh CR), and the pod webhook resolves the -# endpoint synchronously at admission, so these are short. The values give -# headroom for the initial APIReader cache warm-up and the leader-election -# lease the External-reconcile path inherits from the C2 reconciler loop. -EXTERNAL_BACKEND_TIMEOUT="${EXTERNAL_BACKEND_TIMEOUT:-30}" # seconds -EXTERNAL_INJECT_TIMEOUT="${EXTERNAL_INJECT_TIMEOUT:-30}" # seconds -CANONICAL_BACKEND_TIMEOUT="${CANONICAL_BACKEND_TIMEOUT:-30}" # seconds - -# Events-only smoke tunable. An events-only backend provisions no workload, so -# the only wait is the reconciler latching status.firstAvailableAt and the -# KV-event gate publishing Ready=False/AwaitingFirstKVEvent — a sub-second -# server-less reconcile; the budget covers APIReader warm-up + leader-election. -EVENTSONLY_BACKEND_TIMEOUT="${EVENTSONLY_BACKEND_TIMEOUT:-30}" # seconds -HICACHE_SMOKE_TIMEOUT="${HICACHE_SMOKE_TIMEOUT:-30}" # seconds - -# Kernel-check smoke tunables (assertions 14 + 15). -# KERNEL_CHECK_SMOKE_NS is a dedicated namespace created + deleted by those -# two phases so they don't leave fixtures in other namespaces. -KERNEL_CHECK_SMOKE_NS="${KERNEL_CHECK_SMOKE_NS:-ic-smoke-kernel-check}" -# Budget for the report-only engine pod to become Ready (init container runs -# python:3.11-slim; the image pull dominates on a cold node but is small). -KERNEL_CHECK_POD_TIMEOUT="${KERNEL_CHECK_POD_TIMEOUT:-120}" -# Budget for the C2 reconciler to read the init-container termination message -# and publish EngineKernelsHealthy=False. One reconcile cycle + poll buffer. -KERNEL_CHECK_COND_TIMEOUT="${KERNEL_CHECK_COND_TIMEOUT:-60}" -# Legacy-IP paired-binding smoke tunables. The current paired sample contributes -# only the engine scaffold; the legacy CacheBackend fixture is inline. -# -# Default namespace is dedicated to this smoke so re-runs against an existing -# cluster (KEEP_CLUSTER=1) don't mutate or delete a developer's own resources -# in `default`. The script creates the namespace on entry and deletes it on -# the way out. -SAMPLE_NS="${SAMPLE_NS:-cb-engine-smoke}" -POLICY_SMOKE_NS="${POLICY_SMOKE_NS:-ic-smoke-policy}" -PROMPT_TOPOLOGY_SMOKE_NS="${PROMPT_TOPOLOGY_SMOKE_NS:-ic-smoke-prompt-topology}" -# CacheBackend reconciler publishes status.endpoint once the managed -# lmcache-server Service is created — typically within ~5s. 60s absorbs -# cold-start jitter. -SAMPLE_ENDPOINT_TIMEOUT="${SAMPLE_ENDPOINT_TIMEOUT:-60}" -# Reconciler runs initial CacheBackend reconcile + first Matched refresh -# within a few seconds of CB Create. 60s absorbs cold-start jitter. -SAMPLE_MATCH_TIMEOUT="${SAMPLE_MATCH_TIMEOUT:-60}" -# Drift case waits for the 30s self-RequeueAfter cadence to fire after the -# engine pod is gone. 75s = one full cadence + buffer for the patch + pod- -# terminate round-trip. -SAMPLE_DRIFT_TIMEOUT="${SAMPLE_DRIFT_TIMEOUT:-75}" -# Cache-server restart cascade. Each wait covers a different leg of -# the loop: the controller observing the replacement cache-server pod -# and computing its server-instance identifier -# (`:`), then patching the engine Deployment's -# pod template annotations. 60s absorbs the cache-server pod's -# recreate-and-Ready cycle (the busybox stand-in starts in a few -# seconds; the wait dominates on a cold node). -SAMPLE_CASCADE_TIMEOUT="${SAMPLE_CASCADE_TIMEOUT:-60}" -# KV-event gate: time budget for the managed cache-server Deployment to pull -# its image and reach Available, then for the gate to publish -# AwaitingFirstKVEvent. The image pull dominates on a cold node, hence the -# larger default than the other sample waits. -SAMPLE_GATE_TIMEOUT="${SAMPLE_GATE_TIMEOUT:-240}" -# Tiny stand-in for the vLLM image. The webhook injects on pod CREATE; the -# engine doesn't need to run for the operator-facing signals (Matched, -# annotation, Event) to materialize. Avoids a multi-GB pull in CI. -SAMPLE_ENGINE_IMAGE="${SAMPLE_ENGINE_IMAGE:-busybox:1.36}" -# Tiny stand-in for the managed LMCache server image. The controller still -# renders the canonical lmcache_server command/args and TCP readiness probe; the -# image only provides a local binary that listens on the requested port so the -# Deployment can become Available without pulling lmcache/standalone:v0.4.7. -SAMPLE_CACHE_SERVER_IMAGE="${SAMPLE_CACHE_SERVER_IMAGE:-install-smoke-lmcache-server:$TAG}" - -# Image refs match the Makefile's REGISTRY/repo defaults so `kustomize edit set -# image` (or the sed fallback) rewrites the in-tree controller=/server= entries -# without changing their registry/repo paths. -REGISTRY="${REGISTRY:-ghcr.io/cachebox-project}" CONTROLLER_IMG="$REGISTRY/inference-cache-controller:$TAG" SERVER_IMG="$REGISTRY/inference-cache-server:$TAG" - REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" cd "$REPO_ROOT" -# External-backend smoke fixture identifiers. Declared up front so the -# diagnostics helper can reference them even if the smoke aborts before -# the External section creates the objects. -EXT_SMOKE_NS="${EXT_SMOKE_NS:-ic-smoke-external}" -EXT_SMOKE_CB_NAME="cachebackend-external" -EXT_SMOKE_POD_NAME="${EXT_SMOKE_POD_NAME:-smoke-engine}" - -# Canonical hierarchy fixtures: one host-only and one explicitly managed -# provider in the same disposable namespace. -CANONICAL_SMOKE_NS="${CANONICAL_SMOKE_NS:-ic-smoke-canonical-cache}" -CANONICAL_HOST_ONLY_CB="cachebackend-sglang-host-only" -CANONICAL_REDIS_CB="cachebackend-sglang" -CANONICAL_TYPED_CB="sglang-podlocal-host-only" -CANONICAL_TYPED_POD="sglang-podlocal-admission" -CANONICAL_TYPED_VLLM_CB="vllm-podlocal-host-only" -CANONICAL_TYPED_VLLM_POD="vllm-podlocal-admission" +KIND="${KIND:-}" +if [ -z "$KIND" ]; then + if [ -x "$REPO_ROOT/bin/kind" ]; then + KIND="$REPO_ROOT/bin/kind" + else + KIND="kind" + fi +fi -# Events-only-backend smoke fixture identifiers. Declared up front so the -# diagnostics helper can reference them even if the smoke aborts before the -# events-only section creates the objects. -EVENTSONLY_SMOKE_NS="${EVENTSONLY_SMOKE_NS:-ic-smoke-events-only}" -EVENTSONLY_SMOKE_CB_NAME="${EVENTSONLY_SMOKE_CB_NAME:-cachebackend-events-only}" +log() { printf '[default-install-smoke] %s\n' "$*"; } +fail() { printf '[default-install-smoke] ERROR: %s\n' "$*" >&2; exit 1; } -# Native SGLang HiCache fixture identifiers. This engine-local backend has no -# endpoint or controller-owned workload, so its dedicated namespace should -# contain only the persisted CacheBackend used by the smoke. -HICACHE_SMOKE_NS="${HICACHE_SMOKE_NS:-ic-smoke-sglang-hicache}" -HICACHE_SMOKE_CB_NAME="${HICACHE_SMOKE_CB_NAME:-sglang-hicache}" +for binary in curl docker grpcurl kubectl "$KIND"; do + command -v "$binary" >/dev/null 2>&1 || fail "missing required tool: $binary" +done -KIND="${KIND:-$([ -x ./bin/kind ] && echo ./bin/kind || echo kind)}" -pf_pid="" -http_pf_pid="" +mkdir -p "$LOG_DIR" +created_cluster=0 tmpdir="" -kind_config_file="" - -log() { echo "[install-smoke] $*"; } -fail() { - echo "[install-smoke] FAIL: $*" >&2 - collect_diagnostics || true - exit 1 -} +grpc_pf_pid="" +http_pf_pid="" collect_diagnostics() { - mkdir -p "$LOG_DIR" - log "collecting diagnostics into $LOG_DIR" - kubectl get pods -A -o wide >"$LOG_DIR/pods-all.txt" 2>&1 || true - kubectl -n "$NAMESPACE" describe deployment/inference-cache-controller-manager \ - >"$LOG_DIR/describe-controller.txt" 2>&1 || true - kubectl -n "$NAMESPACE" describe deployment/inference-cache-server \ - >"$LOG_DIR/describe-server.txt" 2>&1 || true - kubectl -n "$NAMESPACE" logs deployment/inference-cache-controller-manager --all-containers --tail=-1 \ - >"$LOG_DIR/logs-controller.txt" 2>&1 || true - kubectl -n "$NAMESPACE" logs deployment/inference-cache-server --all-containers --tail=-1 \ - >"$LOG_DIR/logs-server.txt" 2>&1 || true - kubectl get cacheindex cluster-default -o yaml \ - >"$LOG_DIR/cacheindex.yaml" 2>&1 || true - kubectl get cachetenants -A -o yaml \ - >"$LOG_DIR/cachetenants.yaml" 2>&1 || true - kubectl get cachepolicies -A -o yaml \ - >"$LOG_DIR/cachepolicies.yaml" 2>&1 || true - kubectl get prompttemplates -A -o yaml \ - >"$LOG_DIR/prompttemplates.yaml" 2>&1 || true - kubectl get pdtopologies -A -o yaml \ - >"$LOG_DIR/pdtopologies.yaml" 2>&1 || true - kubectl -n cert-manager get pods -o wide \ - >"$LOG_DIR/cert-manager-pods.txt" 2>&1 || true - # Calico CNI state — the smoke swaps kindnet for Calico so NetworkPolicy is - # enforced, so a stuck pod sandbox or an inert policy usually traces back to - # the CNI. Capture node readiness + calico-node/kube-controllers describe+logs. - # Best-effort (|| true): absent on a pre-Calico abort. - kubectl get nodes -o wide \ - >"$LOG_DIR/nodes.txt" 2>&1 || true - kubectl -n kube-system get pods -l k8s-app=calico-node -o wide \ - >"$LOG_DIR/calico-node-pods.txt" 2>&1 || true - kubectl -n kube-system describe daemonset calico-node \ - >"$LOG_DIR/calico-node-describe.txt" 2>&1 || true - kubectl -n kube-system logs -l k8s-app=calico-node --all-containers --tail=-1 \ - >"$LOG_DIR/calico-node-logs.txt" 2>&1 || true - kubectl -n kube-system describe deployment calico-kube-controllers \ - >"$LOG_DIR/calico-kube-controllers-describe.txt" 2>&1 || true - kubectl -n kube-system logs deployment/calico-kube-controllers --tail=-1 \ - >"$LOG_DIR/calico-kube-controllers-logs.txt" 2>&1 || true - # Paired-sample state — only populated if the sample-smoke phase ran; - # safe (|| true) if it didn't. - kubectl -n "$SAMPLE_NS" get cb -o yaml \ - >"$LOG_DIR/sample-cb.yaml" 2>&1 || true - kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo -o yaml \ - >"$LOG_DIR/sample-pods.yaml" 2>&1 || true - kubectl -n "$SAMPLE_NS" get events.events.k8s.io -o yaml \ - >"$LOG_DIR/sample-events.yaml" 2>&1 || true - kubectl -n "$CANONICAL_SMOKE_NS" get cb -o yaml \ - >"$LOG_DIR/canonical-cachebackends.yaml" 2>&1 || true - kubectl -n "$CANONICAL_SMOKE_NS" get deploy,svc,hpa -o yaml \ - >"$LOG_DIR/canonical-provider-workloads.yaml" 2>&1 || true - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml \ - >"$LOG_DIR/sglang-podlocal-admission.yaml" 2>&1 || true - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml \ - >"$LOG_DIR/vllm-podlocal-admission.yaml" 2>&1 || true - # External-backend smoke artefacts. Best-effort — the CR/pod may not - # exist if the smoke aborted before that section. - kubectl get cb -A -o wide \ - >"$LOG_DIR/cachebackends.txt" 2>&1 || true - kubectl get cb -A -o yaml \ - >"$LOG_DIR/cachebackends.yaml" 2>&1 || true - kubectl get cb -n "$EXT_SMOKE_NS" "$EXT_SMOKE_CB_NAME" -o yaml \ - >"$LOG_DIR/external-cb.yaml" 2>&1 || true - kubectl get pod -n "$EXT_SMOKE_NS" "$EXT_SMOKE_POD_NAME" -o yaml \ - >"$LOG_DIR/external-engine-pod.yaml" 2>&1 || true - kubectl get deploy,svc -n "$EXT_SMOKE_NS" \ - >"$LOG_DIR/external-ns-workloads.txt" 2>&1 || true - # Events-only-backend smoke artefacts. Best-effort — the CR may not exist if - # the smoke aborted before that section. - kubectl get cb -n "$EVENTSONLY_SMOKE_NS" "$EVENTSONLY_SMOKE_CB_NAME" -o yaml \ - >"$LOG_DIR/events-only-cb.yaml" 2>&1 || true - kubectl get deploy,svc -n "$EVENTSONLY_SMOKE_NS" \ - >"$LOG_DIR/events-only-ns-workloads.txt" 2>&1 || true - # Native SGLang HiCache smoke artefacts. Best-effort — the CR may not exist - # if the smoke aborted before that section. - kubectl get cb -n "$HICACHE_SMOKE_NS" "$HICACHE_SMOKE_CB_NAME" -o yaml \ - >"$LOG_DIR/sglang-hicache-cb.yaml" 2>&1 || true - kubectl get deploy,svc,hpa -n "$HICACHE_SMOKE_NS" \ - >"$LOG_DIR/sglang-hicache-ns-workloads.txt" 2>&1 || true - # Kernel-check smoke artefacts. Best-effort — the objects may not - # exist if the smoke aborted before that section. - kubectl get cb -n "$KERNEL_CHECK_SMOKE_NS" -o yaml \ - >"$LOG_DIR/kernel-check-cachebackends.yaml" 2>&1 || true - kubectl get pod -n "$KERNEL_CHECK_SMOKE_NS" -o yaml \ - >"$LOG_DIR/kernel-check-pods.yaml" 2>&1 || true - kubectl get events.events.k8s.io -n "$KERNEL_CHECK_SMOKE_NS" \ - >"$LOG_DIR/kernel-check-events.txt" 2>&1 || true + kubectl get nodes -o wide >"$LOG_DIR/nodes.txt" 2>&1 || true + kubectl -n "$SYSTEM_NAMESPACE" get all -o wide >"$LOG_DIR/system.txt" 2>&1 || true + kubectl -n "$SYSTEM_NAMESPACE" get events --sort-by=.lastTimestamp >"$LOG_DIR/events.txt" 2>&1 || true + kubectl -n "$SYSTEM_NAMESPACE" logs deployment/inference-cache-controller-manager --all-containers >"$LOG_DIR/controller.log" 2>&1 || true } cleanup() { - [ -n "$pf_pid" ] && kill "$pf_pid" 2>/dev/null || true - [ -n "$http_pf_pid" ] && kill "$http_pf_pid" 2>/dev/null || true - [ -n "$tmpdir" ] && rm -rf "$tmpdir" - [ -n "$kind_config_file" ] && rm -f "$kind_config_file" - # Only tear the cluster down if we created it (lets local devs pre-create a - # cluster and re-run the smoke without paying the create cost each time). - if [ "${KEEP_CLUSTER:-0}" != "1" ] && [ "${CREATED_CLUSTER:-0}" = "1" ]; then + [ -z "$grpc_pf_pid" ] || kill "$grpc_pf_pid" >/dev/null 2>&1 || true + [ -z "$http_pf_pid" ] || kill "$http_pf_pid" >/dev/null 2>&1 || true + if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then + rm -rf "$tmpdir" + fi + if [ "$created_cluster" = "1" ] && [ "$KEEP_CLUSTER" != "1" ]; then "$KIND" delete cluster --name "$KIND_CLUSTER" >/dev/null 2>&1 || true fi } -# Catch ANY non-zero exit, not just the ones routed through fail(), and dump -# diagnostics BEFORE cleanup deletes the cluster. Without this, a `set -e` -# abort from an unwrapped command (kubectl apply -k, make image-build, kind -# load, the cert-manager apply) tears the cluster down with no artifact left -# behind -- which is exactly the case (e.g. a missing Namespace resource in -# config/default) this gate is meant to surface. on_exit() { - local rc=$? + rc=$? if [ "$rc" -ne 0 ]; then - collect_diagnostics || true + collect_diagnostics fi cleanup + exit "$rc" } trap on_exit EXIT -# --- prereq checks ---------------------------------------------------------- -for bin in docker kubectl curl grpcurl "$KIND"; do - command -v "$bin" >/dev/null 2>&1 || fail "missing required tool on PATH: $bin" -done - -build_sample_cache_server_image() { - local context - context="$(mktemp -d "$tmpdir/lmcache-server-context.XXXXXX")" - - cat >"$context/lmcache_server" <<'EOF' -#!/bin/sh -port="${2:-65432}" -while true; do - nc -l -p "$port" >/dev/null 2>&1 || sleep 1 -done -EOF - - cat >"$context/Dockerfile" <<'EOF' -FROM busybox:1.36 -COPY lmcache_server /usr/local/bin/lmcache_server -RUN chmod +x /usr/local/bin/lmcache_server -EOF - - log "building lightweight lmcache_server stand-in image=$SAMPLE_CACHE_SERVER_IMAGE" - docker build -t "$SAMPLE_CACHE_SERVER_IMAGE" "$context" - log "loading $SAMPLE_CACHE_SERVER_IMAGE into the kind node" - "$KIND" load docker-image "$SAMPLE_CACHE_SERVER_IMAGE" --name "$KIND_CLUSTER" -} - -# Install Calico as a NetworkPolicy-enforcing CNI, then block until it — and the -# node + CoreDNS it unblocks — are fully Ready. kind's built-in kindnet CNI does -# NOT enforce NetworkPolicy, so with kindnet the server NetworkPolicy is inert -# and the /snapshot + /policy + /probe drop assertions can only ever pass on the -# L7 401 fallback. A half-initialised CNI is the main flakiness risk of this -# swap, so every component is waited on explicitly. -# -# Idempotent: `kubectl apply` and every rollout/wait below is a no-op on a -# cluster that already has Calico Ready, so this is safe to call on the reuse -# path as well as after a fresh create. -# -# Timeout budget: the waits sum to 180+120+90+120 = 510s worst case, kept well -# under the workflow's timeout-minutes so that even a wedged CNI fails with time -# to spare for the exit trap to collect diagnostics before GitHub Actions SIGKILLs -# the job. In practice Calico is Ready in well under a minute; these are ceilings. -install_calico() { - log "installing Calico $CALICO_VERSION (NetworkPolicy-enforcing CNI)" - kubectl apply -f \ - "https://raw.githubusercontent.com/projectcalico/calico/$CALICO_VERSION/manifests/calico.yaml" - log "waiting for Calico components to become Ready" - # calico-node is the per-node DaemonSet that programs the dataplane and - # enforces NetworkPolicy; calico-kube-controllers is the policy/IPAM - # controller. rollout status blocks until desired == ready for each. - kubectl -n kube-system rollout status daemonset/calico-node --timeout=180s - kubectl -n kube-system rollout status deployment/calico-kube-controllers --timeout=120s - # The node stays NotReady until the CNI is actually programming the dataplane, - # so node-Ready is the authoritative "Calico works" gate (it also backstops the - # DaemonSet rollout racing to a premature 0-desired success). CoreDNS only gets - # a pod IP once the CNI is up, and the probes below resolve the server Service - # through it, so gate on CoreDNS too. - kubectl wait --for=condition=Ready nodes --all --timeout=90s - kubectl -n kube-system rollout status deployment/coredns --timeout=120s - log "Calico is Ready; NetworkPolicy enforcement is active" -} - -# --- cluster ---------------------------------------------------------------- -# The default-install smoke requires a NetworkPolicy-ENFORCING CNI so the server -# NetworkPolicy actually drops unauthenticated traffic to the server's :8081 -# controller-facing listener (/snapshot, /policy, /probe) — see install_calico -# above and the tightened assertions later in this script. This CNI -# swap is scoped to CI: the human-facing operator reference cluster -# (docs/reference-stack/kind/cluster.yaml) is intentionally left on kindnet — it -# demos the substrate and does not need NetworkPolicy enforcement. if "$KIND" get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER"; then - log "reusing existing kind cluster $KIND_CLUSTER" - CREATED_CLUSTER=0 + log "reusing kind cluster $KIND_CLUSTER" else - log "creating kind cluster $KIND_CLUSTER with the default CNI disabled (Calico installed below)" - # disableDefaultCNI drops kindnet; podSubnet matches Calico's default IPv4 pool - # (192.168.0.0/16) so calico-node hands out addresses from the same range - # kube-controller-manager allocates node podCIDRs from — the canonical - # kind+Calico pairing, no IP-pool patching needed. No `--wait` here: with the - # default CNI disabled the node stays NotReady until Calico is up, so readiness - # is waited on inside install_calico below. - kind_config_file="$(mktemp)" - cat >"$kind_config_file" <<'EOF' -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -networking: - disableDefaultCNI: true - podSubnet: "192.168.0.0/16" -EOF - "$KIND" create cluster --name "$KIND_CLUSTER" --config "$kind_config_file" - rm -f "$kind_config_file" - kind_config_file="" - CREATED_CLUSTER=1 + log "creating kind cluster $KIND_CLUSTER" + "$KIND" create cluster --name "$KIND_CLUSTER" --wait 120s + created_cluster=1 fi kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null -# Guard the reuse path (CREATED_CLUSTER=0). A reused cluster (KEEP_CLUSTER=1) must -# ALREADY be on Calico ALONE, configured the way the create path above sets it up -# — one created by this script is (default CNI disabled + 192.168.0.0/16 pool). -# Anything else enforces NetworkPolicy unreliably and would give the tightened -# drop probes false results, so verify the actual CNI and pod CIDR and bail with -# recreate guidance rather than layering Calico on top / silently degrading: -# 1. kindnet gone — a pure-kindnet cluster OR the dual-CNI state of kindnet with -# Calico added on top both enforce unreliably (checking calico-node presence -# alone would wave the dual-CNI case through); -# 2. calico-node present — an enforcing CNI actually exists; -# 3. node pod CIDR is 192.168.0.0/16 — a stale cluster on kindnet's default -# 10.244.0.0/16 would mismatch Calico's pool. -if [ "$CREATED_CLUSTER" = "0" ]; then - reuse_fix="Delete it ('$KIND delete cluster --name $KIND_CLUSTER') and re-run so the script recreates it with the default CNI disabled + the 192.168.0.0/16 pod CIDR, or unset KEEP_CLUSTER." - if kubectl -n kube-system get daemonset kindnet >/dev/null 2>&1; then - fail "reused kind cluster $KIND_CLUSTER still has the kindnet CNI (kindnet DaemonSet present in kube-system) — this smoke needs Calico as the SOLE NetworkPolicy-enforcing CNI; a kindnet-only or kindnet+Calico cluster enforces unreliably. $reuse_fix" - fi - if ! kubectl -n kube-system get daemonset calico-node >/dev/null 2>&1; then - fail "reused kind cluster $KIND_CLUSTER has no calico-node DaemonSet in kube-system — no NetworkPolicy-enforcing CNI is installed. $reuse_fix" - fi - reuse_pod_cidr="$(kubectl get nodes -o jsonpath='{.items[0].spec.podCIDR}' 2>/dev/null || true)" - case "$reuse_pod_cidr" in - 192.168.*) ;; - *) fail "reused kind cluster $KIND_CLUSTER has pod CIDR '$reuse_pod_cidr', not the 192.168.0.0/16 Calico pool this smoke configures. $reuse_fix" ;; - esac -fi - -# Install (or verify) Calico. install_calico is idempotent, so on a reused -# Calico cluster the apply is a no-op and the readiness waits return -# immediately; on a freshly-created cluster it brings the CNI up. Running it on -# both paths keeps the tightened /snapshot + /policy + /probe drop probes from -# ever executing against a non-enforcing CNI. -install_calico - -# --- build + load images ---------------------------------------------------- -log "building controller + server images at TAG=$TAG" +log "building and loading controller/server images" make image-build TAG="$TAG" REGISTRY="$REGISTRY" - -log "loading $CONTROLLER_IMG into the kind node" "$KIND" load docker-image "$CONTROLLER_IMG" --name "$KIND_CLUSTER" -log "loading $SERVER_IMG into the kind node" "$KIND" load docker-image "$SERVER_IMG" --name "$KIND_CLUSTER" -# --- cert-manager ----------------------------------------------------------- log "installing cert-manager $CERT_MANAGER_VERSION" -kubectl apply -f \ - "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" +kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout=180s -# --- render config/default with SHA-tagged images -------------------------- -# Don't mutate the tracked kustomization.yaml -- copy the whole config tree into -# a tmpdir and edit there. Prefer `kustomize edit set image` when the binary is -# on PATH; sed fallback (scoped to each `- name:` block) keeps the script -# self-contained on a fresh laptop without the kustomize CLI installed. tmpdir="$(mktemp -d)" -cp -r config "$tmpdir/config" -escaped_sample_cache_server_image="$(printf '%s' "$SAMPLE_CACHE_SERVER_IMAGE" | sed 's/[&|\\]/\\&/g')" -manager_manifest="$tmpdir/config/manager/manager.yaml" -if ! grep -q '^ - --lmcache-server-image=lmcache/standalone:v0.4.7$' "$manager_manifest"; then - fail "fixture: config/manager/manager.yaml no longer carries the pinned --lmcache-server-image baseline" -fi -sed -i.bak \ - "s|^ - --lmcache-server-image=lmcache/standalone:v0.4.7$| - --lmcache-server-image=$escaped_sample_cache_server_image|" \ - "$manager_manifest" -rm -f "${manager_manifest}.bak" +cp -R config "$tmpdir/config" ( cd "$tmpdir/config/default" if command -v kustomize >/dev/null 2>&1; then - kustomize edit set image \ - "controller=$CONTROLLER_IMG" \ - "server=$SERVER_IMG" + kustomize edit set image "controller=$CONTROLLER_IMG" "server=$SERVER_IMG" else - # Each `- name: …` block is followed by `newName:` + `newTag:`. Split on - # the LAST `:` so registry-with-port refs (host:port/repo:tag) keep their - # registry path; `${X%:*}` strips the shortest suffix from the final `:`. sed -i.bak \ - -e "/^- name: controller$/,/^- name: server$/ { - s|^ newName: .*| newName: ${CONTROLLER_IMG%:*}| - s|^ newTag: .*| newTag: ${CONTROLLER_IMG##*:}| - }" \ - -e "/^- name: server$/,\$ { - s|^ newName: .*| newName: ${SERVER_IMG%:*}| - s|^ newTag: .*| newTag: ${SERVER_IMG##*:}| - }" \ + -e "/^- name: controller$/,/^- name: server$/ { s|^ newName: .*| newName: ${CONTROLLER_IMG%:*}|; s|^ newTag: .*| newTag: ${CONTROLLER_IMG##*:}|; }" \ + -e "/^- name: server$/,$ { s|^ newName: .*| newName: ${SERVER_IMG%:*}|; s|^ newTag: .*| newTag: ${SERVER_IMG##*:}|; }" \ kustomization.yaml + rm -f kustomization.yaml.bak fi ) -# --- apply + wait ----------------------------------------------------------- -log "applying config/default (controller + server + CRDs + RBAC + webhook)" +log "installing config/default" kubectl apply -k "$tmpdir/config/default" +kubectl -n "$SYSTEM_NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ + deployment/inference-cache-controller-manager deployment/inference-cache-server -log "waiting up to $READY_TIMEOUT for controller + server deployments to reach Available" -kubectl -n "$NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ - deployment/inference-cache-controller-manager \ - deployment/inference-cache-server \ - || fail "controller and/or server did not reach Available within $READY_TIMEOUT" - -controller_args="$(kubectl -n "$NAMESPACE" get deployment/inference-cache-controller-manager \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].args}' 2>/dev/null || true)" -case "$controller_args" in - *"--lmcache-server-image=$SAMPLE_CACHE_SERVER_IMAGE"*) ;; - *) - kubectl -n "$NAMESPACE" get deployment/inference-cache-controller-manager -o yaml || true - fail "controller args do not include --lmcache-server-image=$SAMPLE_CACHE_SERVER_IMAGE: $controller_args" - ;; -esac -log "controller LMCache server image flag=$SAMPLE_CACHE_SERVER_IMAGE" - -# --- server resources sized for DefaultMaxEntries --------------------------- -# The default install MUST budget enough memory to actually hold the -# DefaultMaxEntries=1,000,000 cap; without that, the default cap is a -# meaningless number — operators would OOM well before reaching it. The -# sizing-guide measurements (docs/operations/index-sizing.md) put 1M -# entries at ~540 MiB peak RSS, so the limit lives at 1Gi. Asserting on -# the live Deployment proves the bundle still ships that resource shape — -# the smoke would catch a future refactor that "simplified" the limit -# back to its old 256Mi value, which would silently re-introduce the -# OOM-below-cap discrepancy. -server_mem_limit=$(kubectl -n "$NAMESPACE" get deployment/inference-cache-server \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="server")].resources.limits.memory}' \ - 2>/dev/null || true) -# Normalize to bytes so the assertion catches semantic drift, not literal-string -# drift: K8s quantities like "1Gi" and "1024Mi" are equivalent and either is a -# valid way to express the documented 1 GiB. The minimum sized to fit the -# DefaultMaxEntries=1M cap at ~540 MiB peak RSS plus a 1.5x headroom margin is -# 1 GiB = 1073741824 bytes; we accept anything >= that. -mem_to_bytes() { - # Strips a K8s memory quantity suffix (Ki/Mi/Gi/Ti or k/M/G/T) and emits bytes. - # Returns 0 on unparseable input — caller treats 0 as "below threshold" and - # fails noisily. Uses awk for the multiply so fractional quantities like - # 1.5Gi don't trip bash integer arithmetic (which would crash the gate - # instead of failing it cleanly). - local v="$1" n factor - case "$v" in - *Ki) n=${v%Ki}; factor=1024 ;; - *Mi) n=${v%Mi}; factor=$((1024 * 1024)) ;; - *Gi) n=${v%Gi}; factor=$((1024 * 1024 * 1024)) ;; - *Ti) n=${v%Ti}; factor=$((1024 * 1024 * 1024 * 1024)) ;; - *k) n=${v%k}; factor=1000 ;; - *M) n=${v%M}; factor=$((1000 * 1000)) ;; - *G) n=${v%G}; factor=$((1000 * 1000 * 1000)) ;; - *T) n=${v%T}; factor=$((1000 * 1000 * 1000 * 1000)) ;; - *) n=$v; factor=1 ;; - esac - awk -v n="$n" -v f="$factor" 'BEGIN { - # awk parses leading numerics; "garbage" becomes 0, "1.5" stays 1.5. - # printf "%.0f" rounds the product back to an integer byte count. - printf "%.0f\n", n * f - }' -} -server_mem_bytes=$(mem_to_bytes "$server_mem_limit") -min_bytes=$(( 1024 * 1024 * 1024 )) # 1 GiB -if [ "$server_mem_bytes" -lt "$min_bytes" ]; then - fail "inference-cache-server memory limit = '$server_mem_limit' ($server_mem_bytes bytes); want >= 1Gi ($min_bytes bytes) to fit DefaultMaxEntries=1M per docs/operations/index-sizing.md" -fi -log "inference-cache-server memory limit = $server_mem_limit ($server_mem_bytes bytes; >= 1Gi → sized for DefaultMaxEntries=1M)" - -# --- CacheBackend CRD schema-trim assertion -------------------------------- -# The installed CRD must reflect the inert-field trim: the five removed fields -# are absent from the served v1alpha1 schema, and the field that replaced the -# removed status.indexEntries — status.indexParticipation.prefixCount — is -# present. Probing the live CRD in the cluster (not just the repo manifest) -# proves the trimmed schema is what actually got installed by `kubectl apply -# -k`. Each probe asks for a field's `.type`: absent fields yield empty output, -# present fields yield their OpenAPI type — unambiguous and free of -# map-formatting quirks. -crd_field_type() { - # $1 = jsonpath under the v1alpha1 openAPIV3Schema.properties root - kubectl get crd cachebackends.inferencecache.io \ - -o "jsonpath={.spec.versions[?(@.name=='v1alpha1')].schema.openAPIV3Schema.properties.$1.type}" \ - 2>/dev/null || true -} -if [ -n "$(crd_field_type 'spec.properties.integration.properties.lookupTimeoutMs')" ]; then - fail "CRD still serves removed spec.integration.lookupTimeoutMs (schema trim not installed)" -fi -if [ -n "$(crd_field_type 'spec.properties.integration.properties.minimumPrefixTokens')" ]; then - fail "CRD still serves removed spec.integration.minimumPrefixTokens (schema trim not installed)" -fi -if [ -n "$(crd_field_type 'status.properties.indexEntries')" ]; then - fail "CRD still serves removed status.indexEntries (schema trim not installed)" -fi -# status.indexParticipation.prefixCount is the authoritative count surface that -# replaced the removed status.indexEntries — assert the replacement is served. -if [ -z "$(crd_field_type 'status.properties.indexParticipation.properties.prefixCount')" ]; then - fail "CRD is missing status.indexParticipation.prefixCount (the replacement for status.indexEntries)" -fi -# status.indexParticipation.t2HitRate is the tier-2 (LMCache) offload health -# surface — assert the new status field is actually served by the installed CRD. -if [ -z "$(crd_field_type 'status.properties.indexParticipation.properties.t2HitRate')" ]; then - fail "CRD is missing status.indexParticipation.t2HitRate (the tier-2 health surface)" -fi -# spec.storage{,.pvc} + status.capacity were removed in the storage-retirement -# trim — the lm:// server we provision is in-memory, so a local PVC cannot -# honestly back it; durability is a backend choice. Assert the installed CRD no -# longer serves them, so an operator cannot set a storage field the controller -# no longer honors (the operator-facing surface change this smoke must catch). -if [ -n "$(crd_field_type 'spec.properties.storage')" ]; then - fail "CRD still serves removed spec.storage (storage-retirement trim not installed)" -fi -if [ -n "$(crd_field_type 'status.properties.capacity')" ]; then - fail "CRD still serves removed status.capacity (storage-retirement trim not installed)" -fi -log "CacheBackend CRD reflects the schema trim (lookupTimeoutMs/minimumPrefixTokens/indexEntries/storage/capacity absent; indexParticipation.prefixCount + t2HitRate present)" - -# --- CacheIndex poller assertion ------------------------------------------- -# The controller's CacheIndex poller is leader-elected and refreshes on a 30s -# ticker, so a non-empty observedServer within ~60s of Available proves the -# poller acquired the lease, reached the server's /snapshot endpoint, and wrote -# the singleton CR's status. -log "waiting up to ${CACHEINDEX_TIMEOUT}s for cacheindex/cluster-default to be populated" +log "checking CacheIndex polling and server HTTP/gRPC surfaces" deadline=$(($(date +%s) + CACHEINDEX_TIMEOUT)) observed="" until [ -n "$observed" ]; do - observed="$(kubectl get cacheindex cluster-default \ - -o jsonpath='{.status.observedServer}' 2>/dev/null || true)" - if [ -n "$observed" ]; then break; fi - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl get cacheindex cluster-default -o yaml || true - fail "cacheindex/cluster-default.status.observedServer was empty after ${CACHEINDEX_TIMEOUT}s" - fi + observed="$(kubectl get cacheindex cluster-default -o jsonpath='{.status.observedServer}' 2>/dev/null || true)" + [ -n "$observed" ] && break + [ "$(date +%s)" -lt "$deadline" ] || fail "CacheIndex poller did not publish status.observedServer" sleep 3 done -log "cacheindex/cluster-default.status.observedServer=$observed" -# The CacheIndex table is the operator-facing view for the status poller. Check -# the installed CRD renders the Prefixes/Changed columns and that their JSONPath -# targets actually populate cells in the row instead of falling back to a -# header-only/default NAME/AGE table. -ci_table="$(kubectl get ci cluster-default 2>/dev/null || true)" -ci_header="$(printf '%s\n' "$ci_table" | sed -n '1p')" -ci_row="$(printf '%s\n' "$ci_table" | sed -n '2p')" -for column in PREFIXES CHANGED; do - if ! grep -Eq "(^|[[:space:]])${column}([[:space:]]|$)" <<<"$ci_header"; then - echo "$ci_table" - fail "expected CacheIndex printer column ${column} in kubectl get output" - fi -done -if ! grep -Fq "cluster-default" <<<"$ci_row"; then - echo "$ci_table" - fail "expected CacheIndex printer row to include cluster-default" -fi -ci_prefixes="$(awk 'NR==2 {print $2}' <<<"$ci_table")" -ci_changed="$(awk 'NR==2 {print $3}' <<<"$ci_table")" -if [ "$ci_prefixes" != "0" ]; then - echo "$ci_table" - fail "expected CacheIndex printer column Prefixes=0 in kubectl get output, got: ${ci_prefixes:-}" -fi -if [ -z "$ci_changed" ] || [ "$ci_changed" = "" ] || [ "$ci_changed" = "" ]; then - echo "$ci_table" - fail "expected CacheIndex printer column Changed to be populated in kubectl get output" -fi -log "CacheIndex printer columns render Prefixes=$ci_prefixes Changed=$ci_changed" - -# --- CacheIndex CRD per-tenant-memory deprecation assertion ---------------- -# Per-tenant memory cannot be honestly attributed on a shared, tenant-unaware -# engine (status.tenants[].memoryUsed double-counts the same bytes once per -# tenant), so the field is DEPRECATED and always 0 — but retained in the -# v1alpha1 schema for wire/shape compatibility (removal deferred to v1beta1). -# The honest per-replica engine total stays. Probing the live CRD proves both -# fields are in the installed bundle, not just the repo. -ci_field_type() { - kubectl get crd cacheindices.inferencecache.io \ - -o "jsonpath={.spec.versions[?(@.name=='v1alpha1')].schema.openAPIV3Schema.properties.$1.type}" \ - 2>/dev/null || true -} -if [ -z "$(ci_field_type 'status.properties.tenants.items.properties.memoryUsed')" ]; then - fail "CRD is missing status.tenants[].memoryUsed (deprecated+zeroed but retained in the v1alpha1 schema for compat — must remain until v1beta1)" -fi -if [ -z "$(ci_field_type 'status.properties.replicas.items.properties.cacheMemoryBytes')" ]; then - fail "CRD is missing status.replicas[].cacheMemoryBytes (the honest per-replica engine total)" -fi -log "CacheIndex CRD serves deprecated status.tenants[].memoryUsed (retained, always 0) and the honest status.replicas[].cacheMemoryBytes" - -# --- CacheIndex harmonized-pointer status fields --------------------------- -# hitRate (status.replicas[] and status.tenants[]) and status.tenants[].indexEntries -# use the "nil = not yet reported / computed" pointer convention, aligned with -# the per-instance CacheBackend/CacheTenant surfaces. Pointer-ness itself is NOT -# visible in the OpenAPI schema (a *string still serves as type: string, a -# *int64 as type: integer), so this check only proves the fields still exist -# with their expected scalar leaf types in the installed bundle — the guard is -# against an accidental field drop/rename or a codegen change that alters the -# served type. The value-level nil-vs-observed-0 behavior is exercised by the -# envtest suite (persisted-shape assertions), not here. -if [ "$(ci_field_type 'status.properties.replicas.items.properties.hitRate')" != "string" ]; then - fail "CacheIndex CRD status.replicas[].hitRate is not served as type string" -fi -if [ "$(ci_field_type 'status.properties.tenants.items.properties.hitRate')" != "string" ]; then - fail "CacheIndex CRD status.tenants[].hitRate is not served as type string" -fi -if [ "$(ci_field_type 'status.properties.tenants.items.properties.indexEntries')" != "integer" ]; then - fail "CacheIndex CRD status.tenants[].indexEntries is not served as type integer" -fi -log "CacheIndex CRD serves status.{replicas,tenants}[].hitRate (string) + status.tenants[].indexEntries (integer)" - -# --- CachePolicy push + printer-column setup -------------------------------- -# Apply a CachePolicy in a dedicated namespace and verify its operator-facing -# table columns render. The gRPC side-effect assertion below proves this CR -# was pushed through the controller's authenticated /policy bridge and adopted -# by the server; keeping the apply here gives the watch-triggered reconcile -# time to run before the port-forward opens. -log "resetting CachePolicy smoke namespace $POLICY_SMOKE_NS" -kubectl delete namespace "$POLICY_SMOKE_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null \ - || fail "timed out waiting for prior CachePolicy smoke namespace $POLICY_SMOKE_NS to delete" -log "applying CachePolicy sample in namespace $POLICY_SMOKE_NS" -kubectl create namespace "$POLICY_SMOKE_NS" --dry-run=client -o yaml \ - | kubectl apply -f - >/dev/null -kubectl -n "$POLICY_SMOKE_NS" apply -f config/samples/cache_v1alpha1_cachepolicy.yaml >/dev/null - -# The Eviction printer column is the operator-facing surface kept on the -# CachePolicy CRD. Verify the header AND the row value — the sample -# intentionally omits spec.eviction so this also exercises the -# +kubebuilder:default=LRU marker (the default must fill the column). -cp_table="$(kubectl -n "$POLICY_SMOKE_NS" get cachepolicy cachepolicy-sample 2>/dev/null || true)" -cp_header="$(printf '%s\n' "$cp_table" | sed -n '1p')" -if ! grep -Eq "(^|[[:space:]])EVICTION([[:space:]]|$)" <<<"$cp_header"; then - echo "$cp_table" - fail "expected CachePolicy printer column EVICTION in kubectl get output" -fi -cp_eviction="$(kubectl -n "$POLICY_SMOKE_NS" get cachepolicy cachepolicy-sample \ - -o jsonpath='{.spec.eviction}' 2>/dev/null || true)" -if [ "$cp_eviction" != "LRU" ]; then - echo "$cp_table" - fail "expected .spec.eviction=LRU after the kubebuilder default fired; got '$cp_eviction'" -fi -if ! grep -Fq "cachepolicy-sample" <<<"$cp_table" || \ - ! grep -Fq "LRU" <<<"$cp_table"; then - echo "$cp_table" - fail "expected CachePolicy printer row to include name and Eviction=LRU" -fi -log "CachePolicy default eviction=LRU stamped, printer column renders Eviction" - -# --- CachePolicy admission rejection (one-per-namespace webhook) ------------ -# The installed validating webhook — served through the bundle's webhook -# Service with the cert-manager-injected CA bundle — must reject a SECOND -# CachePolicy in the namespace. Proving it on the real install (not just -# envtest) exercises the Service + cert + CA-injection path an operator's -# `kubectl apply` actually traverses: a broken cainjection annotation, a wrong -# Service selector, or a missing cert would fail here while envtest still -# passed. cachepolicy-sample already occupies $POLICY_SMOKE_NS, so this apply -# must be denied. -log "asserting a second CachePolicy in $POLICY_SMOKE_NS is rejected at admission" -second_cp_yaml="$(cat <&1)"; then - echo "$cp_reject_out" - fail "second CachePolicy in $POLICY_SMOKE_NS was admitted; the one-per-namespace webhook did not fire on the real install" -fi -if ! grep -q "already has CachePolicy" <<<"$cp_reject_out"; then - echo "$cp_reject_out" - fail "second CachePolicy was rejected, but not by the expected webhook rule (missing 'already has CachePolicy' message)" -fi -log "second CachePolicy rejected at admission by the installed validating webhook" - -# --- CacheTenant status projection assertion ------------------------------- -# Apply a CacheTenant and prove the poller's per-tenant projection writes its -# status. The smoke cluster has no engine pods, so the tenant holds zero -# prefixes: the projection must report indexEntries=0 (observed-zero, not nil) -# with Ready=True. This exercises the CacheTenant CRD schema, the combined -# CachePolicy+CacheTenant push to /policy, and the per-tenant status writer. -log "applying CacheTenant sample and waiting for its status projection" -kubectl apply -f config/samples/cache_v1alpha1_cachetenant.yaml -deadline=$(($(date +%s) + CACHEINDEX_TIMEOUT)) -ct_entries="" -until [ -n "$ct_entries" ]; do - ct_entries="$(kubectl get cachetenant cachetenant-sample \ - -o jsonpath='{.status.indexEntries}' 2>/dev/null || true)" - if [ -n "$ct_entries" ]; then break; fi - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl get cachetenant cachetenant-sample -o yaml || true - fail "cachetenant-sample.status.indexEntries was empty after ${CACHEINDEX_TIMEOUT}s" - fi - sleep 3 -done -if [ "$ct_entries" != "0" ]; then - kubectl get cachetenant cachetenant-sample -o yaml || true - fail "expected cachetenant-sample.status.indexEntries=0 (no traffic), got: $ct_entries" -fi -ct_ready="$(kubectl get cachetenant cachetenant-sample \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" -if [ "$ct_ready" != "True" ]; then - kubectl get cachetenant cachetenant-sample -o yaml || true - fail "expected cachetenant-sample Ready=True, got: ${ct_ready:-}" -fi - -# The printer columns (Tenant / Entries / Quota) are themselves an operator- -# facing surface. Verify `kubectl get cachetenants` renders them — a default -# table with only NAME/AGE would mean the additionalPrinterColumns regressed. -ct_table="$(kubectl get cachetenant cachetenant-sample 2>/dev/null || true)" -if ! grep -q 'tenant-a' <<<"$ct_table" || ! grep -q '100000' <<<"$ct_table"; then - echo "$ct_table" - fail "expected CacheTenant printer columns (Tenant=tenant-a, Quota=100000) in kubectl get output" -fi -log "cachetenant-sample.status: indexEntries=$ct_entries Ready=$ct_ready (printer columns OK)" - -# --- CacheTenant admission rejection (tenantID-uniqueness webhook) ---------- -# The installed validating webhook must reject a SECOND CacheTenant claiming an -# already-used tenantID in the same namespace. cachetenant-sample (tenantID -# tenant-a) was applied to the default namespace above, so a second tenant -# reusing tenant-a there must be denied — proving the in-cluster -# Service/cert/CA-injection path for this webhook too. -log "asserting a duplicate-tenantID CacheTenant in the default namespace is rejected at admission" -second_ct_yaml="$(cat <<'EOF' -apiVersion: inferencecache.io/v1alpha1 -kind: CacheTenant -metadata: - name: cachetenant-sample-2 -spec: - tenantID: tenant-a -EOF -)" -if ct_reject_out="$(printf '%s\n' "$second_ct_yaml" | kubectl apply -f - 2>&1)"; then - echo "$ct_reject_out" - fail "second CacheTenant reusing tenantID tenant-a was admitted; the uniqueness webhook did not fire on the real install" -fi -if ! grep -q "already claimed by CacheTenant" <<<"$ct_reject_out"; then - echo "$ct_reject_out" - fail "duplicate CacheTenant was rejected, but not by the expected webhook rule (missing 'already claimed by CacheTenant' message)" -fi -log "duplicate-tenantID CacheTenant rejected at admission by the installed validating webhook" - -# --- CacheTenant admission: reserved probe tenantID ------------------------ -# The functional self-test uses tenant_id "inferencecache.io/probe" as -# server-internal state. The CacheTenant admission webhook MUST reject any -# CR claiming that id so an operator-created tenant cannot collide with the -# probe scope (which would bypass quota enforcement at the PolicyStore -# layer and share the probe's reserved replica). The rule fires on CREATE, -# and on an UPDATE that newly introduces the id — pinned end-to-end by -# this assertion against the real installed webhook (not just envtest). -log "asserting a CacheTenant claiming the reserved probe tenantID is rejected at admission" -reserved_ct_yaml="$(cat <<'EOF' -apiVersion: inferencecache.io/v1alpha1 -kind: CacheTenant -metadata: - name: cachetenant-reserved-probe -spec: - tenantID: inferencecache.io/probe -EOF -)" -if reserved_ct_out="$(printf '%s\n' "$reserved_ct_yaml" | kubectl apply -f - 2>&1)"; then - echo "$reserved_ct_out" - fail "CacheTenant claiming the reserved probe tenantID was admitted; the reservation rule did not fire on the real install" -fi -if ! grep -q "reserved" <<<"$reserved_ct_out"; then - echo "$reserved_ct_out" - fail "reserved-probe-tenantID CacheTenant was rejected, but not by the expected rule (missing 'reserved' in the diagnostic)" -fi -log "CacheTenant claiming the reserved probe tenantID rejected by the installed validating webhook" - -# --- PromptTemplate + PDTopology schema-only assertion ---------------------- -# config/default installs the PromptTemplate/PDTopology CRDs and RBAC, and the -# controller manager adds their Go types to the scheme, but no PromptTemplate -# render-controller or PDTopology reconciler is started from cmd/controller/main.go. -# Their status fields are therefore future status surfaces, not live signals in -# config/default. Assert the meaningful Phase-1 contract instead: committed -# samples apply against the real CRDs, and the short-name kubectl tables expose -# their operator-facing printer columns. -log "applying PromptTemplate + PDTopology samples in namespace $PROMPT_TOPOLOGY_SMOKE_NS" -kubectl delete namespace "$PROMPT_TOPOLOGY_SMOKE_NS" --ignore-not-found --wait=true --timeout=60s >/dev/null \ - || fail "timed out waiting for prior PromptTemplate/PDTopology smoke namespace $PROMPT_TOPOLOGY_SMOKE_NS to delete" -kubectl create namespace "$PROMPT_TOPOLOGY_SMOKE_NS" --dry-run=client -o yaml \ - | kubectl apply -f - >/dev/null -kubectl -n "$PROMPT_TOPOLOGY_SMOKE_NS" apply -f config/samples/cache_v1alpha1_prompttemplate.yaml >/dev/null -kubectl -n "$PROMPT_TOPOLOGY_SMOKE_NS" apply -f config/samples/cache_v1alpha1_pdtopology.yaml >/dev/null - -pt_table="$(kubectl -n "$PROMPT_TOPOLOGY_SMOKE_NS" get pt prompttemplate-sample 2>/dev/null || true)" -pt_header="$(printf '%s\n' "$pt_table" | sed -n '1p')" -pt_row="$(printf '%s\n' "$pt_table" | sed -n '2p')" -if ! grep -Eq "(^|[[:space:]])REVISION([[:space:]]|$)" <<<"$pt_header"; then - echo "$pt_table" - fail "expected PromptTemplate printer column REVISION in kubectl get pt output" -fi -if ! grep -Fq "prompttemplate-sample" <<<"$pt_row"; then - echo "$pt_table" - fail "expected PromptTemplate printer row to include prompttemplate-sample" -fi - -pdt_table="$(kubectl -n "$PROMPT_TOPOLOGY_SMOKE_NS" get pdt pdtopology-sample 2>/dev/null || true)" -pdt_header="$(printf '%s\n' "$pdt_table" | sed -n '1p')" -pdt_row="$(printf '%s\n' "$pdt_table" | sed -n '2p')" -for column in PREFILL DECODE; do - if ! grep -Eq "(^|[[:space:]])${column}([[:space:]]|$)" <<<"$pdt_header"; then - echo "$pdt_table" - fail "expected PDTopology printer column ${column} in kubectl get pdt output" - fi -done -if ! grep -Fq "pdtopology-sample" <<<"$pdt_row" || \ - ! grep -Fq "prefill-a" <<<"$pdt_row" || \ - ! grep -Fq "decode-a" <<<"$pdt_row"; then - echo "$pdt_table" - fail "expected PDTopology printer row to include sample name and prefill/decode pool names" -fi -log "PromptTemplate/PDTopology are schema-only in default install; samples apply and printer columns render" -kubectl delete namespace "$PROMPT_TOPOLOGY_SMOKE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true - -# --- gRPC fail-open assertion ---------------------------------------------- -log "port-forwarding svc/inference-cache-server :9090 -> localhost:$GRPC_LOCAL_PORT" -mkdir -p "$LOG_DIR" -kubectl -n "$NAMESPACE" port-forward svc/inference-cache-server "$GRPC_LOCAL_PORT:9090" \ - >"$LOG_DIR/port-forward.log" 2>&1 & -pf_pid=$! - -# Wait for the local port to accept connections. config/default serves :9090 -# PLAINTEXT (TLS is opt-in — exercised separately against the -# config/overlays/server-tls overlay near the end of this smoke), so probe with -# -plaintext. +kubectl -n "$SYSTEM_NAMESPACE" port-forward svc/inference-cache-server "$GRPC_LOCAL_PORT:9090" >"$LOG_DIR/grpc-port-forward.log" 2>&1 & +grpc_pf_pid=$! +kubectl -n "$SYSTEM_NAMESPACE" port-forward svc/inference-cache-server "$HTTP_LOCAL_PORT:8080" >"$LOG_DIR/http-port-forward.log" 2>&1 & +http_pf_pid=$! for _ in $(seq 1 30); do - if grpcurl -plaintext -max-time 2 "localhost:$GRPC_LOCAL_PORT" list >/dev/null 2>&1; then - break - fi + grpcurl -plaintext -max-time 2 "localhost:$GRPC_LOCAL_PORT" list >/dev/null 2>&1 && break sleep 1 done - -# --- server HTTP surface assertion ----------------------------------------- -# The installed Service also exposes the operator/observability HTTP surface on -# :8080. Probe it through kubectl port-forward so this covers Service wiring, -# the real process listeners, readiness, and Prometheus registration together. -log "port-forwarding svc/inference-cache-server :8080 -> localhost:$HTTP_LOCAL_PORT" -kubectl -n "$NAMESPACE" port-forward svc/inference-cache-server "$HTTP_LOCAL_PORT:8080" \ - >"$LOG_DIR/port-forward-http.log" 2>&1 & -http_pf_pid=$! - -http_ready=0 +grpcurl -plaintext -max-time 5 "localhost:$GRPC_LOCAL_PORT" list >"$LOG_DIR/grpc-services.txt" \ + || fail "default plaintext gRPC endpoint is unavailable" +lookup_response="$(grpcurl -plaintext -max-time 5 \ + -import-path proto -proto inferencecache/v1alpha1/inferencecache.proto \ + -d '{"modelId":"install-smoke-unknown"}' "localhost:$GRPC_LOCAL_PORT" \ + inferencecache.v1alpha1.InferenceCache/LookupRoute)" \ + || fail "LookupRoute smoke request failed" +grep -Eq '"(reasonCode|reason_code)"[[:space:]]*:[[:space:]]*"NO_HINT"' <<<"$lookup_response" \ + || fail "LookupRoute did not fail open with NO_HINT: $lookup_response" for _ in $(seq 1 30); do - if curl -fsS --max-time 2 "http://localhost:$HTTP_LOCAL_PORT/readyz" \ - >"$LOG_DIR/readyz.out" 2>"$LOG_DIR/readyz.err"; then - http_ready=1 - break - fi + curl -fsS --max-time 2 "http://localhost:$HTTP_LOCAL_PORT/readyz" >/dev/null 2>&1 && break sleep 1 done -if [ "$http_ready" != "1" ]; then - cat "$LOG_DIR/port-forward-http.log" >&2 || true - cat "$LOG_DIR/readyz.err" >&2 || true - fail "server /readyz did not return 200 on :8080 within 30s" -fi - -if ! curl -fsS --max-time 5 "http://localhost:$HTTP_LOCAL_PORT/metrics" \ - >"$LOG_DIR/metrics.out" 2>"$LOG_DIR/metrics.err"; then - cat "$LOG_DIR/port-forward-http.log" >&2 || true - cat "$LOG_DIR/metrics.err" >&2 || true - fail "server /metrics did not return 200 on :8080" -fi -if ! grep -Eq '^inferencecache_server_up[[:space:]]+1([[:space:]]|$)' "$LOG_DIR/metrics.out"; then - grep -n 'inferencecache_server_up' "$LOG_DIR/metrics.out" >&2 || true - fail "server /metrics did not expose inferencecache_server_up 1" -fi -log "server HTTP surface OK: /readyz returned 200 and /metrics exposes inferencecache_server_up 1" - -# --- functional-probe gate: positive-case coverage scoped to a later phase -- -# The controller-side caller publishes the FunctionalProbeOK condition on -# managed CacheBackends. This smoke asserts the NEGATIVE case directly -# (FunctionalProbeOK absent while the upstream KV-event gate holds -# Ready=False) inline in the paired-sample phase below, alongside the -# KV-event gate assertion it cascades off of — see the -# "functional-probe gate (downstream of KV-event gate)" block after the -# AwaitingFirstKVEvent assertion. The positive case (FunctionalProbeOK -# appearing once the upstream gate clears) requires an engine workload -# that actually publishes KV events, which this smoke does not stand up; -# that assertion lands with a follow-up that ships an engine-pod fixture. -# The metric inferencecache_backend_probe_result_total is similarly not -# /metrics-visible on this install because Prometheus client_golang's -# CounterVec exposes no HELP/TYPE/data lines until a WithLabelValues -# child is instantiated, and that requires a real probe call to fire — -# which only happens after the upstream gate clears. Stage 2's -# controller-side wiring is otherwise covered by 20+ unit tests and an -# envtest integration sub-test driving a real CacheBackend reconciler -# against an httptest /probe server. - -# --- gRPC default posture assertion (plaintext) ---------------------------- -# config/default serves :9090 plaintext. Assert a plaintext client can list the -# services. (We deliberately do NOT probe with a TLS client here: a TLS -# ClientHello against the plaintext HTTP/2 listener destabilizes the shared -# kubectl port-forward and breaks the probes that follow. The TLS-rejected-on- -# plaintext direction is covered by the unit tests; the opt-in TLS phase near -# the end proves the inverse — plaintext rejected once TLS is enabled.) -log "asserting gRPC default posture on :9090 (plaintext serving)" -if ! grpcurl -plaintext -max-time 5 "localhost:$GRPC_LOCAL_PORT" list \ - >"$LOG_DIR/grpcurl-plaintext-list.out" 2>&1; then - cat "$LOG_DIR/grpcurl-plaintext-list.out" >&2 || true - fail "expected plaintext 'grpcurl -plaintext list' to succeed against the default (plaintext) install" -fi -log "gRPC default posture OK: plaintext serving" - -# Probe twice in priority order: server reflection first (what a real gateway -# client would do), proto-file second (survives the reflection registration -# being temporarily reverted or moved). The default install is plaintext, so -# these functional probes use -plaintext; TLS is verified in the opt-in phase. -grpcurl_lookup_route() { - local payload="$1" - local err_file="${2:-$LOG_DIR/grpcurl.err}" - if grpcurl -plaintext -max-time 5 -d "$payload" \ - "localhost:$GRPC_LOCAL_PORT" \ - inferencecache.v1alpha1.InferenceCache/LookupRoute \ - 2>"$err_file"; then - return 0 - fi - log "reflection LookupRoute probe failed; falling back to proto-file probe" >&2 - grpcurl -plaintext -max-time 5 \ - -import-path proto -proto inferencecache/v1alpha1/inferencecache.proto \ - -d "$payload" \ - "localhost:$GRPC_LOCAL_PORT" \ - inferencecache.v1alpha1.InferenceCache/LookupRoute \ - 2>>"$err_file" -} - -grpcurl_report_cache_state() { - local payload="$1" - local err_file="${2:-$LOG_DIR/grpcurl-report-cache-state.err}" - # Default install is plaintext (see grpcurl_lookup_route). - if printf '%s\n' "$payload" | grpcurl -plaintext -max-time 5 -d @ \ - "localhost:$GRPC_LOCAL_PORT" \ - inferencecache.v1alpha1.InferenceCache/ReportCacheState \ - 2>"$err_file"; then - return 0 - fi - log "reflection ReportCacheState probe failed; falling back to proto-file probe" >&2 - printf '%s\n' "$payload" | grpcurl -plaintext -max-time 5 \ - -import-path proto -proto inferencecache/v1alpha1/inferencecache.proto \ - -d @ \ - "localhost:$GRPC_LOCAL_PORT" \ - inferencecache.v1alpha1.InferenceCache/ReportCacheState \ - 2>>"$err_file" -} - -has_reason_code() { - local resp="$1" - local want="$2" - grep -Eq "\"(reasonCode|reason_code)\"[[:space:]]*:[[:space:]]*\"$want\"" <<<"$resp" -} - -probe_lookup_route() { - local payload='{"modelId":"install-smoke-unknown"}' - grpcurl_lookup_route "$payload" "$LOG_DIR/grpcurl.err" -} - -resp="$(probe_lookup_route)" || { - cat "$LOG_DIR/grpcurl.err" >&2 || true - fail "grpcurl LookupRoute did not return a response" -} -log "LookupRoute response: $resp" - -# JSON field name varies between gRPC encoders (camelCase reasonCode from -# grpcurl's default JSONPB, snake_case reason_code in the .proto). Accept -# either form; require the value to be NO_HINT (the documented fail-open code -# for an unknown model). -if ! has_reason_code "$resp" "NO_HINT"; then - fail "expected reason_code=NO_HINT for an unknown model, got: $resp" -fi - -# --- Dual-input LookupRoute assertions (server-side tokenization surface) --- -# CONTRIBUTING requires gRPC behavior changes to extend this gate. The default -# install ships the pure-Go server (no cgo tokenizer), so prove BOTH dual-input -# paths are wired at the deployment level and fail open: -# - token_ids: the server fingerprints the supplied tokens itself (no tokenizer -# needed); for an unknown model it fails open to NO_HINT. -# - prompt_text: the default build has no tokenizer, so this path fails open to -# NO_HINT (never an error on the hot path). -ti_resp="$(grpcurl_lookup_route '{"modelId":"install-smoke-unknown","hashScheme":"vllm","tokenIds":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]}' "$LOG_DIR/grpcurl-tokenids.err")" || { - cat "$LOG_DIR/grpcurl-tokenids.err" >&2 || true - fail "grpcurl LookupRoute(token_ids) did not return a response" -} -log "LookupRoute(token_ids) response: $ti_resp" -if ! has_reason_code "$ti_resp" "NO_HINT"; then - fail "expected reason_code=NO_HINT for token_ids on an unknown model, got: $ti_resp" -fi - -pt_resp="$(grpcurl_lookup_route '{"modelId":"install-smoke-unknown","hashScheme":"vllm","promptText":"hello world"}' "$LOG_DIR/grpcurl-prompttext.err")" || { - cat "$LOG_DIR/grpcurl-prompttext.err" >&2 || true - fail "grpcurl LookupRoute(prompt_text) did not return a response" -} -log "LookupRoute(prompt_text) response: $pt_resp" -if ! has_reason_code "$pt_resp" "NO_HINT"; then - fail "expected reason_code=NO_HINT for prompt_text on the default (tokenizer-less) build, got: $pt_resp" -fi - -# --- CachePolicy PUSH adoption assertion ----------------------------------- -# No read-back endpoint exists for /policy, by design. Prove the server adopted -# the controller-pushed CachePolicy via existing gRPC side effects on two -# orthogonal axes (routingFloorScore is probed separately by the patch-and- -# wait block further below): -# 1. minimumPrefixTokens (request-side gate; applied BEFORE the index -# lookup under affinityRouting=Disabled or as a post-lookup -# result-side downgrade under affinityRouting=Enabled — the default). -# Seed one prefix; the request below the policy threshold no longer -# reaches the prefix-match path. With affinityRouting=Enabled (the -# kubebuilder default carried by the sample CR) the fallback fires and -# the response is AFFINITY_HINT — *not* PREFIX_MATCH, which is what -# proves the gate adopted. With affinityRouting=Disabled this same path -# would return NO_HINT; either non-PREFIX_MATCH outcome proves the gate. -# 2. minimumMatchedTokens (result-side floor, applied AFTER the lookup, against -# the realized matched-token overlap). Seed a SECOND prefix whose stored -# tokenCount is above minimumPrefixTokens (so it clears the request-side -# gate) but BELOW minimumMatchedTokens (so the realized match downgrades -# away from PREFIX_MATCH). Without this assertion the floor could be -# silently dropped and the smoke would still pass on the request-side -# gate alone. The sample carries minimumMatchedTokens explicitly -# (config/samples/cache_v1alpha1_cachepolicy.yaml). The downgrade -# again surfaces as AFFINITY_HINT (or NO_HINT under affinity Disabled). -# -# Note: with no CachePolicy at all the server-wide DefaultMinimumMatchedTokens -# (= 64) ALSO downgrades the trivial 32-token match away from PREFIX_MATCH — -# the no-policy fallback fires the same floor as the sample CR sets. The -# point of the trivial-match assertion is therefore "the pushed CR did not -# silently drop the result-side floor" rather than "without the CR this would -# have been PREFIX_MATCH". The low-prefix lookup is the standalone proof -# that policy adoption happened (its non-PREFIX_MATCH outcome IS owned by -# the pushed minimumPrefixTokens: 32 — no-policy would have ungated the -# request and returned PREFIX_MATCH on the 64-token stored prefix). Together -# they cover both policy enforcement axes end-to-end. Avoids engine -# pods/images, model traffic, and any new transport. -log "seeding two prefixes and asserting CachePolicy minimumPrefixTokens (request-side gate) AND minimumMatchedTokens (result-side floor) are both enforced by LookupRoute" -policy_model="install-smoke-policy" -policy_replica="policy-smoke-replica" -policy_hash_b64="cG9saWN5LXByZWZpeA==" # base64("policy-prefix") — stored at tokenCount=64, clears both gates -trivial_hash_b64="dHJpdmlhbC1wcmVmaXg=" # base64("trivial-prefix") — stored at tokenCount=32, clears request gate (32 >= sample's 32) but below the 64 matched-tokens floor - -# Two stored prefixes in one ReportCacheState ingest: the regular 64-token -# prefix (clears both gates) and the trivial 32-token prefix (clears the -# request-side gate, fails the result-side floor — what proves the -# minimumMatchedTokens floor actually fires end-to-end). -policy_report_payload="$(cat <&2 || true - fail "grpcurl ReportCacheState did not accept the CachePolicy smoke prefixes" -} -log "ReportCacheState response: $policy_report_resp" +curl -fsS --max-time 5 "http://localhost:$HTTP_LOCAL_PORT/readyz" >/dev/null \ + || fail "server /readyz is unavailable" +curl -fsS --max-time 5 "http://localhost:$HTTP_LOCAL_PORT/metrics" >"$LOG_DIR/server-metrics.txt" \ + || fail "server /metrics is unavailable" +grep -Eq '^inferencecache_server_up[[:space:]]+1([[:space:]]|$)' "$LOG_DIR/server-metrics.txt" \ + || fail "server up metric is missing" -# Three lookups exercise the three orthogonal policy-enforcement paths: -# - policy_high: above both gates → PREFIX_MATCH (control: ingest path -# works; affinity does NOT preempt a real match). -# - policy_low: below the request-side gate → AFFINITY_HINT (request-side -# gate fires; the affinity fallback then picks the only known replica). -# - policy_trivial: above the request-side gate but matched_tokens=32 < -# floor 64 → AFFINITY_HINT (result-side floor fires — the assertion the -# request-side gate alone cannot make — and again the affinity fallback -# surfaces the single known replica). -policy_high_payload="$(cat <&2 - echo "$policy_high_resp" >&2 - echo "below-request-gate LookupRoute response (want AFFINITY_HINT — request gate fires; affinity fallback picks the known replica):" >&2 - echo "$policy_low_resp" >&2 - echo "trivial-match (below result floor) LookupRoute response (want AFFINITY_HINT — matched-tokens floor fires; affinity fallback picks the known replica):" >&2 - echo "$policy_trivial_resp" >&2 - for err_file in "$LOG_DIR/grpcurl-policy-high.err" "$LOG_DIR/grpcurl-policy-low.err" "$LOG_DIR/grpcurl-policy-trivial.err"; do - if [ -s "$err_file" ]; then - echo "$(basename "$err_file"):" >&2 - cat "$err_file" >&2 - fi - done - fail "server did not adopt the pushed CachePolicy within ${POLICY_PUSH_TIMEOUT}s (want above PREFIX_MATCH, below-request AFFINITY_HINT, trivial-match AFFINITY_HINT — the non-PREFIX_MATCH outcomes prove both gates fired with affinityRouting=Enabled)" - fi - sleep 2 -done -log "CachePolicy push adopted: above-threshold lookup hit; below-request-gate lookup returned AFFINITY_HINT; trivial-match (matched_tokens&2 || true - fail "grpcurl ReportCacheState did not accept the adapter-scoped smoke prefix" -} -log "ReportCacheState (adapter-scoped) response: $adapter_report_resp" - -adapter_same_payload="$(cat <&2 - echo "$adapter_same_resp" >&2 - if [ -s "$LOG_DIR/grpcurl-adapter-same.err" ]; then - cat "$LOG_DIR/grpcurl-adapter-same.err" >&2 - fi - fail "LookupRoute with the ingesting adapter_id did not return PREFIX_MATCH within ${POLICY_PUSH_TIMEOUT}s — the adapter partition must not break matching inside it" +log "checking the served CacheBackend schema is MP-only" +crd_yaml="$(kubectl get crd cachebackends.inferencecache.io -o yaml)" +for retired in LMCacheServer LMCacheConnectorV1 LMCACHE_REMOTE_URL 'lm://' workerImage workerPort remoteSerde hostMemory observedServerInstance deploymentKind autoscaling; do + if grep -Fq "$retired" <<<"$crd_yaml"; then + fail "served CacheBackend CRD still contains retired surface: $retired" fi - sleep 2 done -# assert_adapter_no_alias runs one NEGATIVE adapter probe. The property under -# test is "the identical prefix hash does NOT match in a different partition," -# but a transport error mustn't masquerade as that: grpcurl failing or returning -# an empty body would leave has_reason_code false and silently pass. So require -# the RPC to SUCCEED and return a non-empty body, then assert the expected -# fail-open reason_code — AFFINITY_HINT, because affinityRouting is Enabled for -# this tenant (the policy block above proved it) and the replica serves the -# adapter-blind (tenant, model, hash_scheme) scope, so a non-matching adapter -# partition downgrades to a stable affinity pick rather than a prefix hit. An -# RPC error is therefore a red, not a green. -assert_adapter_no_alias() { - local payload="$1" err_file="$2" label="$3" alias_msg="$4" - local resp - if ! resp="$(grpcurl_lookup_route "$payload" "$err_file")" || [ -z "$resp" ]; then - if [ -s "$err_file" ]; then - cat "$err_file" >&2 - fi - echo "$label LookupRoute response: ${resp:-}" >&2 - fail "$label LookupRoute returned no response — an RPC/transport error must fail the adapter-partition probe, not silently pass it as a non-match" - fi - if has_reason_code "$resp" "PREFIX_MATCH"; then - echo "$label LookupRoute response (want NOT PREFIX_MATCH):" >&2 - echo "$resp" >&2 - fail "$alias_msg" - fi - if ! has_reason_code "$resp" "AFFINITY_HINT"; then - echo "$label LookupRoute response (want fail-open AFFINITY_HINT):" >&2 - echo "$resp" >&2 - fail "$label LookupRoute did not return the expected fail-open AFFINITY_HINT — a non-matching adapter partition must downgrade to a stable affinity pick, never error and never PREFIX_MATCH" - fi -} - -assert_adapter_no_alias "$adapter_other_payload" "$LOG_DIR/grpcurl-adapter-other.err" \ - "different-adapter" \ - "LookupRoute for adapter_id=smoke-lora-b matched a prefix ingested under smoke-lora-a — identical token content aliased across adapters, which would route to a replica holding a DIFFERENT adapter's KV" -assert_adapter_no_alias "$adapter_none_payload" "$LOG_DIR/grpcurl-adapter-none.err" \ - "no-adapter" \ - "LookupRoute without adapter_id matched an adapter-scoped prefix — adapter-scoped ingest must stay out of the default partition" -log "adapter partition enforced end-to-end: same adapter_id → PREFIX_MATCH; a different adapter_id and an absent adapter_id each returned the fail-open AFFINITY_HINT (RPC succeeded, off the prefix-match path) on the identical prefix hash" - -# --- routingFloorScore end-to-end probe ------------------------------------ -# Proves the new field flows CR → controller flatten → /policy push → server -# resolver → buildLookupResponse downgrade. The same 64-token prefix that -# returned PREFIX_MATCH above is forced off the prefix-match path after we -# patch the live CachePolicy to routingFloorScore="1000" (well above any -# plausible score: matched_tokens (64) × freshness (~1) × -# distinguishing_power (1.0 — only one replica was seeded, so the factor -# degenerates) = ~64, which is below the strict 1000 floor). With the -# kubebuilder-default affinityRouting=Enabled the response settles at -# AFFINITY_HINT (the floor fired and the affinity fallback picked the -# single known replica). Restoring "0.1" must flip the response back to -# PREFIX_MATCH — proving replace-on-write semantics carry the new field -# too. Without engine pods or model traffic, this is the minimum end-to-end -# exercise of the new operator-facing knob. -log "asserting CachePolicy.spec.routingFloorScore is propagated and gates LookupRoute" - -apply_floor() { - local floor="$1" - kubectl -n "$POLICY_SMOKE_NS" patch cachepolicy cachepolicy-sample \ - --type=merge -p "{\"spec\":{\"routingFloorScore\":\"$floor\"}}" >/dev/null \ - || fail "kubectl patch cachepolicy routingFloorScore=$floor failed" -} - -wait_floor_reason() { - local payload="$1" want="$2" err_file="$3" label="$4" - local deadline=$(($(date +%s) + POLICY_PUSH_TIMEOUT)) - local resp="" - until has_reason_code "$resp" "$want"; do - resp="$(grpcurl_lookup_route "$payload" "$err_file")" || true - if has_reason_code "$resp" "$want"; then - break - fi - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$POLICY_SMOKE_NS" get cachepolicy cachepolicy-sample -o yaml || true - echo "$label response (want $want):" >&2 - echo "$resp" >&2 - if [ -s "$err_file" ]; then - echo "$(basename "$err_file"):" >&2 - cat "$err_file" >&2 - fi - fail "server did not adopt routingFloorScore patch within ${POLICY_PUSH_TIMEOUT}s ($label, want $want)" - fi - sleep 2 - done -} - -apply_floor "1000" -wait_floor_reason "$policy_high_payload" "AFFINITY_HINT" "$LOG_DIR/grpcurl-policy-floor-strict.err" "routingFloorScore=1000 high-token lookup" -log "routingFloorScore=1000 enforced: same 64-token match now AFFINITY_HINT (score below floor; affinity fallback picks the known replica)" - -apply_floor "0.1" -wait_floor_reason "$policy_high_payload" "PREFIX_MATCH" "$LOG_DIR/grpcurl-policy-floor-restored.err" "routingFloorScore=0.1 high-token lookup" -log "routingFloorScore=0.1 restored: same 64-token match flipped back to PREFIX_MATCH (replace-on-write OK)" +kubectl create namespace "$SMOKE_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -kubectl delete namespace "$POLICY_SMOKE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true - -# --- legacy IP paired-binding compatibility smoke -------------------------- -# INTENTIONAL LEGACY FIXTURE: this section exercises the LMCacheServer/IP -# implementation retained until Phase 7. It is not a production reference. -# The current config/samples/cachebackend-with-engine.yaml supplies only the -# inference-system-owned engine scaffold; the legacy CacheBackend is written -# inline below. The current typed MP path is exercised in the canonical section. -# -# Asserts the -# CacheBackend ↔ engine-pod binding's operator-facing signals materialize -# end-to-end: -# - status.matchedEnginePods → 1 -# - inferencecache.io/injected-by annotation on the engine pod -# - InjectedByCacheBackend Event on the engine pod whose regarding.uid -# equals the persisted pod's metadata.uid (so the Event surfaces -# under `kubectl describe pod`, not just `kubectl get events`) -# Then exercises the drift case: scale the engine Deployment to 0, force- -# delete the (terminating) pod, and assert status.matchedEnginePods → 0 -# via the reconciler's RequeueAfter cadence within SAMPLE_DRIFT_TIMEOUT. -# -# Engine-side image is swapped to SAMPLE_ENGINE_IMAGE (busybox by default) -# BEFORE the engine Deployment lands so the sample exercises the wiring -# without paying a multi-GB vLLM pull. The webhook injects on pod CREATE; -# the signals here all materialize from object state, not from the engine -# actually running. -# -# Two-step apply (CB first, wait for status.endpoint, then engine -# Deployment) avoids a race the webhook would otherwise fail-open -# through: if the engine pod's admission lands before the reconciler has -# published the CacheBackend's status.endpoint, the webhook admits the -# pod unmodified (no annotation, no Event) — the rest of the smoke -# would then assert against a pod that's missing the signals through no -# product fault. Pre-publishing the endpoint closes the race. -log "creating sample namespace $SAMPLE_NS" -kubectl create namespace "$SAMPLE_NS" --dry-run=client -o yaml \ - | kubectl apply -f - >/dev/null - -log "splitting the current paired sample to reuse its engine scaffold" -sample_file="config/samples/cachebackend-with-engine.yaml" -# Place the split files under the trapped $tmpdir so an early failure -# between split and apply (or a SIGINT mid-run) does not leak temp files -# in /tmp. $tmpdir is created at script init and removed by cleanup(). -sample_tmp_cb="$(mktemp "$tmpdir/sample-cb.XXXXXX")" -sample_tmp_engine="$(mktemp "$tmpdir/sample-engine.XXXXXX")" -# The paired sample is a two-doc YAML stream (CacheBackend, ---, -# Deployment) — guaranteed ordering, so awk on the `---` separator is -# enough. yq would be cleaner but isn't a guaranteed dependency on the -# CI image. -awk -v cb="$sample_tmp_cb" -v engine="$sample_tmp_engine" ' - /^---$/ { sep=1; next } - !sep { print > cb } - sep { print > engine } -' "$sample_file" - -# Replace the typed MP CacheBackend document with an explicitly isolated legacy -# fixture. Keeping it inline prevents a repository-owned sample from presenting -# LMCacheServer as a supported deployment choice. -cat >"$sample_tmp_cb" <<'EOF' +log "creating typed PodLocal MP backends" +cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply -f - >/dev/null apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: - name: qwen-demo-cache + name: host-only spec: runtime: VLLM type: LMCache - deploymentKind: Deployment - replicas: 1 + engineSelector: + matchLabels: + app: mp-engine integration: role: ReadWrite + lmCache: + topology: PodLocal + chunkSizeTokens: 256 + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 1Gi + maxWorkers: 1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + memory: 2Gi +--- +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: managed-redis +spec: + runtime: SGLang + type: LMCache engineSelector: matchLabels: - app: qwen-demo - observation: - modelID: Qwen/Qwen2.5-0.5B-Instruct + app: sglang-engine + integration: + role: ReadWrite + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 1Gi + maxWorkers: 1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + memory: 2Gi remoteStorage: - provider: LMCacheServer + provider: Redis ownership: Managed - lmCacheServer: - image: lmcache/standalone:v0.4.7 + workload: + nodeSelector: + kubernetes.io/os: linux + terminationGracePeriodSeconds: 45 + redis: {} EOF -# Patch the engine container's image to the lightweight stand-in. The -# Deployment's metadata stays untouched (still qwen-engine, still -# labeled app=qwen-demo), so the binding label flow is exercised exactly -# as a real operator would experience it. -sed -i.bak "s|vllm/vllm-openai-cpu:latest-x86_64|$SAMPLE_ENGINE_IMAGE|g" \ - "$sample_tmp_engine" -rm -f "${sample_tmp_engine}.bak" - -build_sample_cache_server_image -if ! grep -q '^ image: lmcache/standalone:v0.4.7$' "$sample_tmp_cb"; then - fail "legacy inline fixture no longer carries remoteStorage.lmCacheServer.image" -fi -sed -i.bak '/^ image: lmcache\/standalone:v0.4.7$/d' "$sample_tmp_cb" -rm -f "${sample_tmp_cb}.bak" -if grep -q '^ image:' "$sample_tmp_cb"; then - fail "fixture: failed to remove the CR-level LMCache image before testing the controller flag" -fi - -log "applying CacheBackend" -kubectl -n "$SAMPLE_NS" apply -f "$sample_tmp_cb" >/dev/null - -log "waiting up to ${SAMPLE_ENDPOINT_TIMEOUT}s for status.endpoint" -deadline=$(($(date +%s) + SAMPLE_ENDPOINT_TIMEOUT)) -endpoint="" -while [ "$(date +%s)" -lt "$deadline" ]; do - endpoint=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true) - if [ -n "$endpoint" ]; then break; fi - sleep 2 -done -if [ -z "$endpoint" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "CacheBackend.status.endpoint not populated after ${SAMPLE_ENDPOINT_TIMEOUT}s" -fi -log "status.endpoint=$endpoint" - -cb_provider_image="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.spec.remoteStorage.lmCacheServer.image}' 2>/dev/null || true)" -if [ -n "$cb_provider_image" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "cb.spec.remoteStorage.lmCacheServer.image=$cb_provider_image, want absent so the controller flag supplies it" -fi -dep_provider_image="$(kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="lmcache-server")].image}' \ - 2>/dev/null || true)" -if [ "$dep_provider_image" != "$SAMPLE_CACHE_SERVER_IMAGE" ]; then - kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache -o yaml || true - fail "deploy.lmcache-server.image=$dep_provider_image, want controller flag image $SAMPLE_CACHE_SERVER_IMAGE" -fi -log "managed LMCache image resolved from controller flag: $dep_provider_image" - -log "applying engine Deployment (image=$SAMPLE_ENGINE_IMAGE)" -kubectl -n "$SAMPLE_NS" apply -f "$sample_tmp_engine" >/dev/null -# Split files live under $tmpdir and are removed by the trap; no -# explicit rm here so a failure between split and apply still cleans up. - -log "waiting up to ${SAMPLE_MATCH_TIMEOUT}s for status.matchedEnginePods=1" -deadline=$(($(date +%s) + SAMPLE_MATCH_TIMEOUT)) -matched="" -while [ "$(date +%s)" -lt "$deadline" ]; do - matched=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.matchedEnginePods}' 2>/dev/null || true) - if [ "$matched" = "1" ]; then break; fi - sleep 2 -done -if [ "$matched" != "1" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo -o wide || true - fail "status.matchedEnginePods=$matched, want 1 after ${SAMPLE_MATCH_TIMEOUT}s" -fi -log "status.matchedEnginePods=1" - -# --- provider resource fallback -------------------------------------------- -# The paired sample uses remoteStorage.lmCacheServer and omits its resources. -# Defaulting stays renderer-local: the provider renderer puts a bounded 4Gi -# request / 8Gi limit on the lmcache-server container without persisting it in -# the CacheBackend. -log "asserting provider resources stay out of the CR and default onto the rendered Deployment" -cb_provider_resources="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.spec.remoteStorage.lmCacheServer.resources}' 2>/dev/null || true)" -if [ -n "$cb_provider_resources" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "cb.spec.remoteStorage.lmCacheServer.resources=$cb_provider_resources, want absent when the sample omits it" -fi -dep_lim_mem="$(kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="lmcache-server")].resources.limits.memory}' \ - 2>/dev/null || true)" -if [ "$dep_lim_mem" != "8Gi" ]; then - kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache -o yaml || true - fail "deploy.lmcache-server.resources.limits.memory=$dep_lim_mem, want 8Gi (canonical provider fallback not applied)" -fi -dep_req_mem="$(kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache \ - -o jsonpath='{.spec.template.spec.containers[?(@.name=="lmcache-server")].resources.requests.memory}' \ - 2>/dev/null || true)" -if [ "$dep_req_mem" != "4Gi" ]; then - kubectl -n "$SAMPLE_NS" get deploy qwen-demo-cache -o yaml || true - fail "deploy.lmcache-server.resources.requests.memory=$dep_req_mem, want 4Gi (canonical provider fallback not applied)" -fi -log "provider resources defaulted on the workload only: requests.memory=4Gi limits.memory=8Gi" - -# --- KV-event readiness gate assertion (operator-facing) -------------------- -# The managed backend has an engine pod attached (matchedEnginePods=1), but the -# smoke's stub engine (busybox) emits no KV events and the controller runs with -# no kvevent-subscriber sidecar, so NO KV event will ever be observed — the -# exact demo-day failure mode the gate exists to surface (engine present, -# KV-event stream silent). We assert the gate's operator-visible surfaces end to -# end on the real install: -# - spec.observation.firstEventTimeout defaulted to 5m (CRD field + -# admission defaulting); -# - once the managed cache-server reaches Available, the gate holds the -# backend at Ready=False / reason AwaitingFirstKVEvent — the deterministic -# condition surface (Ready can never become True here, so without the gate -# the backend would have been reported Ready on rollout); -# - status.firstKVEventObservedAt stays UNSET — the durable latch is written -# the instant a KV event is observed, so its absence is the gate-specific -# "nothing observed" signal. -fet="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.spec.observation.firstEventTimeout}' 2>/dev/null || true)" -# Accept both "5m" (CRD-schema default, applied when the observation block is -# present) and "5m0s" (Go Duration.String(), the webhook-materialized form) — -# both decode to the same 5m duration. -if [ "$fet" != "5m" ] && [ "$fet" != "5m0s" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "spec.observation.firstEventTimeout=$fet, want 5m (CRD default / webhook defaulter not applied)" -fi - -# The gate only evaluates once the managed cache-server Deployment is Available, -# so wait for that first; then the awaited state is deterministic (no events). -log "waiting up to ${SAMPLE_GATE_TIMEOUT}s for the managed cache-server Deployment to reach Available" -if ! kubectl -n "$SAMPLE_NS" wait --for=condition=Available --timeout="${SAMPLE_GATE_TIMEOUT}s" \ - deployment/qwen-demo-cache >/dev/null 2>&1; then - kubectl -n "$SAMPLE_NS" get deployment/qwen-demo-cache -o yaml || true - kubectl -n "$SAMPLE_NS" get pod -l app.kubernetes.io/instance=qwen-demo-cache -o wide || true - fail "managed cache-server Deployment did not reach Available within ${SAMPLE_GATE_TIMEOUT}s; cannot exercise the KV-event gate" -fi -log "waiting up to ${SAMPLE_GATE_TIMEOUT}s for the gate to publish Ready=False / AwaitingFirstKVEvent" -deadline=$(($(date +%s) + SAMPLE_GATE_TIMEOUT)) -gate_status="" -gate_reason="" -while [ "$(date +%s)" -lt "$deadline" ]; do - gate_status="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - gate_reason="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - if [ "$gate_status" = "False" ] && [ "$gate_reason" = "AwaitingFirstKVEvent" ]; then break; fi - sleep 2 +for _ in $(seq 1 60); do + endpoint="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend managed-redis -o jsonpath='{.status.remoteStorage.endpoint}' 2>/dev/null || true)" + [ "$endpoint" = "managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] && break + sleep 1 done -if [ "$gate_status" != "False" ] || [ "$gate_reason" != "AwaitingFirstKVEvent" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "KV-event gate not engaged: Ready=$gate_status/$gate_reason, want False/AwaitingFirstKVEvent (engine attached, no KV events)" -fi - -gate_latch="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.firstKVEventObservedAt}' 2>/dev/null || true)" -if [ -n "$gate_latch" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "status.firstKVEventObservedAt=$gate_latch, want unset (no KV event source exists, so the gate latch must never be written)" -fi -log "KV-event gate engaged: firstEventTimeout=$fet, Ready=False/AwaitingFirstKVEvent, firstKVEventObservedAt unset" - -# --- functional-probe gate (downstream of KV-event gate) ------------------- -# The functional-probe gate is cascade-prevented from running while any -# upstream gate keeps Ready=False — there is no point in driving a -# synthetic round-trip against a backend the operator has not yet -# declared "is supposed to be working." This asserts that -# operator-visible behavior on the live install: -# - The FunctionalProbeOK condition MUST be ABSENT on a backend the -# upstream KV-event gate is holding at Ready=False/AwaitingFirstKVEvent. -# Its presence here would be a regression: the controller is firing -# the probe loop on a backend that's still warming up, paging -# operators on a known-not-ready state. -# - The Ready condition's status+reason still reflect the upstream gate -# (False/AwaitingFirstKVEvent), not a downstream probe verdict — -# proving cascade-prevention is on, not just "no probe call yet." -# A positive-case assertion (FunctionalProbeOK appearing once the -# upstream gate clears) requires an engine workload that actually -# publishes KV events; not feasible from this smoke without a real GPU -# or the CPU vLLM image. That's deferred to a Stage 4 follow-up that -# lands an engine-pod fixture; this negative case still locks the -# operator-facing cascade behavior in place. -fp_status="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="FunctionalProbeOK")].status}' 2>/dev/null || true)" -if [ -n "$fp_status" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "FunctionalProbeOK condition present (status=$fp_status) while Ready=$gate_status/$gate_reason; cascade-prevention regressed — downstream probe gate must not fire while upstream KV-event gate is False" -fi -log "functional-probe cascade-prevention holds: FunctionalProbeOK absent while Ready=False/AwaitingFirstKVEvent" - -# T2Degraded (advisory tier-2 offload health) must likewise be ABSENT on a -# backend that has not exercised its tier-2 cache: the condition is derived from -# status.indexParticipation.t2HitRate, which stays nil until external lookups -# are observed. A fresh smoke backend drives no tier-2 traffic, so the operator -# must NOT see a T2Degraded breadcrumb here — a present condition (even -# False) would be a misleading "tier-2 is being tracked" signal where the -# tier was never used. -t2_status="$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="T2Degraded")].status}' 2>/dev/null || true)" -if [ -n "$t2_status" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - fail "T2Degraded condition present (status=$t2_status) on a backend that has not exercised tier-2 — it must be absent until external lookups are observed" -fi -log "T2Degraded absent until tier-2 is exercised (no-traffic steady state)" +[ "${endpoint:-}" = "managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] || fail "managed Redis endpoint was not published" +kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis >/dev/null +kubectl -n "$SMOKE_NAMESPACE" get service managed-redis >/dev/null +managed_os="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.nodeSelector.kubernetes\.io/os}')" +[ "$managed_os" = "linux" ] || fail "managed workload nodeSelector was not rendered" +managed_grace="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}')" +[ "$managed_grace" = "45" ] || fail "managed workload terminationGracePeriodSeconds was not rendered" + +log "checking real Pod admission renders only the MP wire" +pod_json="$tmpdir/admitted-pod.json" +cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -o json -f - >"$pod_json" +apiVersion: v1 +kind: Pod +metadata: + name: mp-engine + labels: + app: mp-engine +spec: + containers: + - name: vllm + image: busybox:1.36 + command: ["sh", "-c", "sleep 3600"] +EOF -# --- EngineCompatibility (injected-engine crash-loop) assertion ------------- -# ORDERING (do not move earlier): this phase blocks up to ~300s (180s for the -# injected engine to reach CrashLoopBackOff + 120s for the advisory condition to -# publish). It MUST run AFTER the KV-event-gate AwaitingFirstKVEvent assertion -# above, NOT between matchedEnginePods=1 and that gate. The gate's -# AwaitingFirstKVEvent window is bounded by spec.observation.firstEventTimeout -# (defaulted to 5m and asserted as 5m above), anchored at the cache-server -# Deployment becoming Available; the busybox engine never emits KV events, so the -# backend deterministically flips AwaitingFirstKVEvent -> NoKVEventsObserved once -# that 5m elapses. Running this ~300s wait before the gate assertion would burn -# most of that 5m budget and race the gate to NoKVEventsObserved on slow CI -# (flaky). Placed here, the gate is already banked and EngineCompatibility is -# independent of the Ready condition, so the long wait is harmless. -# -# This asserts the controller surfaces an injected engine's CrashLoopBackOff as -# the advisory EngineCompatibility condition — it does NOT validate the -# hybrid-attention incompatibility *cause* (a real hybrid model would need a -# GPU). The condition reports the generic crash-loop observation; the root cause -# is verified out-of-band via engine logs. The busybox stand-in -# (SAMPLE_ENGINE_IMAGE) CANNOT run `vllm serve`, so its injected engine -# container lands in CrashLoopBackOff — which is all this assertion needs: it -# exercises the controller's crash-loop heuristic, not the connector-versus- -# hybrid-attention diagnosis. We must NOT mutate the qwen-engine Deployment -# here: a later phase cascade-restarts it to assert status.observedServerInstance -# advances, and a command override would stick and break that check. So we only -# wait for the natural crash-loop and assert the advisory -# EngineCompatibility=False/InjectedEngineCrashLooping condition surfaces -# (instead of a silent crash-loop). The controller does NOT watch engine pods -# (no informer, by design), so we poke the CacheBackend to drive the reconcile -# that reads them. -log "asserting EngineCompatibility surfaces on the crash-looping injected engine" -deadline=$(($(date +%s) + 180)) -clbo="" -while [ "$(date +%s)" -lt "$deadline" ]; do - clbo=$(kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo \ - -o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.state.waiting.reason} {end}{end}' 2>/dev/null || true) - case "$clbo" in *CrashLoopBackOff*) break;; esac - sleep 4 -done -case "$clbo" in - *CrashLoopBackOff*) : ;; - *) kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo -o wide || true - fail "engine pod did not reach CrashLoopBackOff within 180s (waiting reasons: $clbo)" ;; -esac -deadline=$(($(date +%s) + 120)) -ec_reason="" -ec_status="" -while [ "$(date +%s)" -lt "$deadline" ]; do - kubectl -n "$SAMPLE_NS" annotate cb qwen-demo-cache \ - inferencecache.io/smoke-poke="$(date +%s)" --overwrite >/dev/null 2>&1 || true - ec_reason=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="EngineCompatibility")].reason}' 2>/dev/null || true) - ec_status=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.conditions[?(@.type=="EngineCompatibility")].status}' 2>/dev/null || true) - # Assert BOTH status and reason: the documented surface is - # EngineCompatibility=False/InjectedEngineCrashLooping. Checking the reason - # alone would let a regression to status=True (advisory condition inverted) - # slip through while the reason string still matched. - if [ "$ec_status" = "False" ] && [ "$ec_reason" = "InjectedEngineCrashLooping" ]; then break; fi - sleep 4 +grep -Fq 'lmcache-mp-server' "$pod_json" || fail "native MP sidecar was not injected" +grep -Fq 'LMCacheMPConnector' "$pod_json" || fail "vLLM MP connector was not injected" +grep -Fq 'lmcache.mp.host' "$pod_json" || fail "MP loopback host was not injected" +for retired in LMCacheConnectorV1 LMCACHE_REMOTE_URL LMCACHE_REMOTE_SERDE 'lm://'; do + if grep -Fq "$retired" "$pod_json"; then + fail "admitted Pod contains retired wire: $retired" + fi done -if [ "$ec_status" != "False" ] || [ "$ec_reason" != "InjectedEngineCrashLooping" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o jsonpath='{.status.conditions}' || true - fail "EngineCompatibility status=$ec_status reason=$ec_reason, want False/InjectedEngineCrashLooping after the injected engine crash-looped" -fi -log "EngineCompatibility=False/InjectedEngineCrashLooping surfaced on the crash-looping injected engine" - -# Persisted pod identity (UID is server-assigned post-admission; the -# whole point of the engine-pod-events controller is to record the -# Event with this UID, not the empty one a webhook-recorded Event -# would have). -engine_pod=$(kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) -engine_uid=$(kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo \ - -o jsonpath='{.items[0].metadata.uid}' 2>/dev/null || true) -if [ -z "$engine_pod" ] || [ -z "$engine_uid" ]; then - fail "no engine pod labeled app=qwen-demo found in namespace $SAMPLE_NS" -fi -log "engine pod: $engine_pod (uid=$engine_uid)" -# The mutating webhook stamps the annotation on successful injection. -# Absence here means the webhook either didn't fire or fail-opened. -injected_by=$(kubectl -n "$SAMPLE_NS" get pod "$engine_pod" \ - -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true) -if [ "$injected_by" != "$SAMPLE_NS/qwen-demo-cache" ]; then - fail "annotation inferencecache.io/injected-by=$injected_by, want $SAMPLE_NS/qwen-demo-cache" -fi -log "annotation inferencecache.io/injected-by=$injected_by" - -# Event assertion. Polls because the events.EventRecorder broadcasts -# asynchronously. Match on (regarding.uid, reason) — NOT just name — -# because describe-by-UID is the user-facing contract. -log "waiting for InjectedByCacheBackend Event with regarding.uid=$engine_uid" -deadline=$(($(date +%s) + 30)) -seen="" -while [ "$(date +%s)" -lt "$deadline" ]; do - seen=$(kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=InjectedByCacheBackend \ - -o jsonpath="{range .items[?(@.regarding.uid=='$engine_uid')]}{.reason}{'\n'}{end}" \ - 2>/dev/null || true) - if [ -n "$seen" ]; then break; fi - sleep 2 -done -if [ -z "$seen" ]; then - kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=InjectedByCacheBackend -o yaml || true - fail "InjectedByCacheBackend Event not observed on pod uid=$engine_uid within 30s" -fi -log "InjectedByCacheBackend Event present on the engine pod (UID matches the persisted pod)" - -# --- binding diagnostics: unmatched selector + explicit skip ---------------- -# A deliberately non-matching CacheBackend should expose the selector drift on -# the CR itself: status.engineSelectorMessage echoes the selector, and a Normal -# EngineSelectorUnmatched Event provides the push-style breadcrumb. -log "applying a non-matching CacheBackend to assert selector diagnostics" -cat </dev/null -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: unmatched-cache -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: External - endpoint: unmatched-cache.example.com:8200 - engineSelector: - matchLabels: - app: definitely-not-qwen -EOF - -log "waiting up to ${SAMPLE_MATCH_TIMEOUT}s for unmatched selector status + Event" -deadline=$(($(date +%s) + SAMPLE_MATCH_TIMEOUT)) -unmatched_msg="" -unmatched_event="" -while [ "$(date +%s)" -lt "$deadline" ]; do - unmatched_msg="$(kubectl -n "$SAMPLE_NS" get cb unmatched-cache \ - -o jsonpath='{.status.engineSelectorMessage}' 2>/dev/null || true)" - unmatched_event="$(kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=EngineSelectorUnmatched \ - -o jsonpath="{range .items[?(@.regarding.name=='unmatched-cache')]}{.reason}{'\n'}{end}" \ - 2>/dev/null || true)" - case "$unmatched_msg" in - *"app:definitely-not-qwen"*"no Pods in namespace match"*) - [ -n "$unmatched_event" ] && break - ;; - esac - sleep 2 -done -case "$unmatched_msg" in - *"app:definitely-not-qwen"*"no Pods in namespace match"*) ;; - *) - kubectl -n "$SAMPLE_NS" get cb unmatched-cache -o yaml || true - fail "status.engineSelectorMessage=$unmatched_msg, want selector echo + no Pods diagnostic" - ;; -esac -if [ -z "$unmatched_event" ]; then - kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=EngineSelectorUnmatched -o yaml || true - fail "EngineSelectorUnmatched Event not observed on CacheBackend/unmatched-cache within ${SAMPLE_MATCH_TIMEOUT}s" -fi -log "unmatched selector diagnostics present: $unmatched_msg" - -# A pod that explicitly opts out must be distinguishable from a drifted pod: -# the webhook stamps inject-skipped, and the engine-pod-events controller turns -# that stamp into a describe-visible SkippedByOperator Event keyed to the -# persisted pod UID. -log "creating a skip-inject pod to assert opt-out visibility" -cat </dev/null -apiVersion: v1 -kind: Pod -metadata: - name: skipped-engine - annotations: - inferencecache.io/skip-inject: "true" - labels: - app: skip-demo -spec: - containers: - - name: engine - image: $SAMPLE_ENGINE_IMAGE - command: ["sh", "-c", "sleep 3600"] -EOF -skipped_uid="$(kubectl -n "$SAMPLE_NS" get pod skipped-engine \ - -o jsonpath='{.metadata.uid}' 2>/dev/null || true)" -if [ -z "$skipped_uid" ]; then - fail "skipped-engine pod did not persist with a UID" -fi -deadline=$(($(date +%s) + 30)) -skip_reason="" -skip_event="" -while [ "$(date +%s)" -lt "$deadline" ]; do - skip_reason="$(kubectl -n "$SAMPLE_NS" get pod skipped-engine \ - -o jsonpath='{.metadata.annotations.inferencecache\.io/inject-skipped}' 2>/dev/null || true)" - skip_event="$(kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=SkippedByOperator \ - -o jsonpath="{range .items[?(@.regarding.uid=='$skipped_uid')]}{.reason}{'\n'}{end}" \ - 2>/dev/null || true)" - [ "$skip_reason" = "skip-inject-annotation" ] && [ -n "$skip_event" ] && break - sleep 2 -done -if [ "$skip_reason" != "skip-inject-annotation" ]; then - kubectl -n "$SAMPLE_NS" get pod skipped-engine -o yaml || true - fail "annotation inferencecache.io/inject-skipped=$skip_reason, want skip-inject-annotation" -fi -skipped_injected_by="$(kubectl -n "$SAMPLE_NS" get pod skipped-engine \ - -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true)" -if [ -n "$skipped_injected_by" ]; then - kubectl -n "$SAMPLE_NS" get pod skipped-engine -o yaml || true - fail "skipped pod unexpectedly carries inferencecache.io/injected-by=$skipped_injected_by" -fi -if [ -z "$skip_event" ]; then - kubectl -n "$SAMPLE_NS" get events.events.k8s.io \ - --field-selector reason=SkippedByOperator -o yaml || true - fail "SkippedByOperator Event not observed on skipped pod uid=$skipped_uid within 30s" -fi -log "skip-inject visibility present: inject-skipped=$skip_reason and SkippedByOperator Event" - -# --- cache-server restart cascade ------------------------------------------ -# When the cache-server pod is replaced (OOM-kill, eviction, image roll, -# operator-initiated restart, …), every injected engine pod holds a stale -# LMCache client socket — the upstream LMServerConnector opens its TCP -# socket in __init__ only and silently fails every subsequent PUT with -# EPIPE until the engine pod itself rolls. The controller's -# observedServerInstance latch detects the cache-server UID transition -# and cascade-restarts every engine Deployment that owns pods carrying -# this backend's inferencecache.io/injected-by annotation AND the -# matching inferencecache.io/injected-by-uid (the UID half rejects -# forgeries and stale name-reuse), by patching -# AnnotationCacheServerRestartTrigger onto the Deployment's pod template -# (the same mechanism kubectl rollout restart uses). -# -# This phase asserts the end-to-end loop on the real install: -# - status.observedServerInstance picks up the current cache-server -# server-instance identifier (`:`) after the -# initial rollout (no cascade — first observation never cascades). -# - Forcing a cache-server pod restart flips observedServerInstance -# to the replacement's server-instance identifier. -# - The engine Deployment's spec.template.metadata.annotations gets -# the cascade trigger set to that new identifier — proving the -# loop closed against the installed RBAC + actual apiserver Patch, -# not just envtest. -# -# Must run BEFORE the drift case below, which scales the engine to 0: -# with no engine pod present, no injected-by annotations remain in the -# namespace, so the cascade would find no Deployments to annotate. -log "waiting up to ${SAMPLE_CASCADE_TIMEOUT}s for the initial cache-server pod to publish status.observedServerInstance" -deadline=$(($(date +%s) + SAMPLE_CASCADE_TIMEOUT)) -baseline_server_instance="" -while [ "$(date +%s)" -lt "$deadline" ]; do - baseline_server_instance=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.observedServerInstance}' 2>/dev/null || true) - if [ -n "$baseline_server_instance" ]; then break; fi - sleep 2 -done -if [ -z "$baseline_server_instance" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - kubectl -n "$SAMPLE_NS" get pod -l app.kubernetes.io/instance=qwen-demo-cache -o wide || true - fail "status.observedServerInstance not populated within ${SAMPLE_CASCADE_TIMEOUT}s; cannot exercise the cache-server restart cascade" -fi -log "baseline status.observedServerInstance=$baseline_server_instance" - -# Force-delete the cache-server pod to simulate the OOM / restart trigger. -# The Deployment controller recreates it with a fresh UID. -cache_pod=$(kubectl -n "$SAMPLE_NS" get pod -l app.kubernetes.io/instance=qwen-demo-cache \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) -if [ -z "$cache_pod" ]; then - fail "no cache-server pod labeled app.kubernetes.io/instance=qwen-demo-cache found in $SAMPLE_NS" -fi -log "deleting cache-server pod $cache_pod to simulate restart" -# Don't mask the delete with `|| true` — a failed trigger here would -# silently look like the cascade isn't firing, which would be -# diagnosed as a controller bug instead of a smoke-script failure. -if ! kubectl -n "$SAMPLE_NS" delete pod "$cache_pod" \ - --force --grace-period=0 >/dev/null 2>&1; then - kubectl -n "$SAMPLE_NS" get pod -l app.kubernetes.io/instance=qwen-demo-cache -o wide || true - fail "failed to delete cache-server pod $cache_pod to simulate restart" -fi - -# Wait for the controller to observe the new pod and update the latch. -log "waiting up to ${SAMPLE_CASCADE_TIMEOUT}s for status.observedServerInstance to flip to the replacement's server-instance identifier" -deadline=$(($(date +%s) + SAMPLE_CASCADE_TIMEOUT)) -new_server_instance="" -while [ "$(date +%s)" -lt "$deadline" ]; do - cur=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.observedServerInstance}' 2>/dev/null || true) - if [ -n "$cur" ] && [ "$cur" != "$baseline_server_instance" ]; then - new_server_instance="$cur" - break - fi - sleep 2 -done -if [ -z "$new_server_instance" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - kubectl -n "$SAMPLE_NS" get pod -l app.kubernetes.io/instance=qwen-demo-cache -o wide || true - fail "status.observedServerInstance did not advance past $baseline_server_instance within ${SAMPLE_CASCADE_TIMEOUT}s" -fi -log "status.observedServerInstance flipped: $baseline_server_instance → $new_server_instance" - -# Assert the engine Deployment's pod template carries the cascade trigger -# annotation set to the new server-instance identifier (the same value the -# controller wrote to status.observedServerInstance). The annotation is -# the mechanism that drives the rolling restart, so its absence here is -# a missed cascade. -log "waiting up to ${SAMPLE_CASCADE_TIMEOUT}s for the engine Deployment to receive the cascade trigger annotation" -deadline=$(($(date +%s) + SAMPLE_CASCADE_TIMEOUT)) -trigger="" -while [ "$(date +%s)" -lt "$deadline" ]; do - trigger=$(kubectl -n "$SAMPLE_NS" get deploy qwen-engine \ - -o jsonpath='{.spec.template.metadata.annotations.inferencecache\.io/cache-server-restart-trigger}' \ - 2>/dev/null || true) - if [ "$trigger" = "$new_server_instance" ]; then break; fi - sleep 2 -done -if [ "$trigger" != "$new_server_instance" ]; then - kubectl -n "$SAMPLE_NS" get deploy qwen-engine -o yaml || true - fail "engine Deployment cascade trigger=$trigger, want $new_server_instance (the cache-server restart did not cascade)" -fi -log "engine Deployment qwen-engine carries inferencecache.io/cache-server-restart-trigger=$trigger" - -# --- drift case: cadence-driven Matched → 0 -------------------------------- -# Scale engine to 0; force-delete to avoid the (terminating) pod still -# being label-visible to the reconciler's pod List for an extended -# time when the image is unavailable. Then wait for the next -# RequeueAfter cycle (no CB-side change, no Owned watch event — pure -# cadence) to drive Matched=0. -log "scaling engine Deployment to 0 to exercise the RequeueAfter cadence" -kubectl -n "$SAMPLE_NS" scale deploy qwen-engine --replicas=0 >/dev/null -kubectl -n "$SAMPLE_NS" delete pod "$engine_pod" \ - --force --grace-period=0 >/dev/null 2>&1 || true -log "waiting up to ${SAMPLE_DRIFT_TIMEOUT}s for status.matchedEnginePods=0 via cadence" -deadline=$(($(date +%s) + SAMPLE_DRIFT_TIMEOUT)) -drifted="" -while [ "$(date +%s)" -lt "$deadline" ]; do - drifted=$(kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache \ - -o jsonpath='{.status.matchedEnginePods}' 2>/dev/null || true) - if [ "$drifted" = "0" ]; then break; fi - sleep 2 -done -if [ "$drifted" != "0" ]; then - kubectl -n "$SAMPLE_NS" get cb qwen-demo-cache -o yaml || true - kubectl -n "$SAMPLE_NS" get pod -l app=qwen-demo -o wide || true - fail "status.matchedEnginePods=$drifted after engine scaled to 0; want 0 via cadence within ${SAMPLE_DRIFT_TIMEOUT}s" -fi -log "drift converged: status.matchedEnginePods=0 via the self-RequeueAfter cadence" - -# Sample cleanup: tear down the whole dedicated namespace in one shot -# (best-effort; failure here doesn't fail the smoke). The script created -# the namespace at the start of this phase, so this leaves the cluster -# in the state the rest of the smoke produced. -kubectl delete namespace "$SAMPLE_NS" \ - --wait=false --ignore-not-found=true >/dev/null 2>&1 || true - -# --- Canonical cache hierarchy --------------------------------------------- -# Proves the API distinction this surface promises: -# * no remoteStorage => no provider workload or endpoint; -# * explicit Managed Redis => Redis workload + RESP Service endpoint. -log "exercising canonical host-only and Managed Redis hierarchies in namespace $CANONICAL_SMOKE_NS" -kubectl create namespace "$CANONICAL_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -kubectl -n "$CANONICAL_SMOKE_NS" apply \ - -f config/samples/cachebackend-sglang-host-only.yaml >/dev/null \ - || fail "canonical host-only CacheBackend sample failed to apply" - -deadline=$(($(date +%s) + CANONICAL_BACKEND_TIMEOUT)) -host_observed_generation="" -while [ -z "$host_observed_generation" ] && [ "$(date +%s)" -lt "$deadline" ]; do - host_observed_generation="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_HOST_ONLY_CB" \ - -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true)" - [ -n "$host_observed_generation" ] || sleep 1 -done -host_runtime="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_HOST_ONLY_CB" \ - -o jsonpath='{.spec.runtime}' 2>/dev/null || true)" -host_remote_provider="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_HOST_ONLY_CB" \ - -o jsonpath='{.spec.remoteStorage.provider}' 2>/dev/null || true)" -host_endpoint="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_HOST_ONLY_CB" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" -if [ -z "$host_observed_generation" ] || [ "$host_runtime" != "SGLang" ] || \ - [ -n "$host_remote_provider" ] || \ - [ -n "$host_endpoint" ]; then - kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_HOST_ONLY_CB" -o yaml || true - fail "canonical host-only state is wrong: observedGeneration=$host_observed_generation runtime=$host_runtime provider=$host_remote_provider endpoint=$host_endpoint" -fi -for resource in deployment service horizontalpodautoscaler; do - if kubectl -n "$CANONICAL_SMOKE_NS" get "$resource" "$CANONICAL_HOST_ONLY_CB" >/dev/null 2>&1; then - kubectl -n "$CANONICAL_SMOKE_NS" get deploy,svc,hpa -o wide || true - fail "canonical host-only CacheBackend unexpectedly created $resource/$CANONICAL_HOST_ONLY_CB" - fi -done -log "canonical host-only hierarchy published no provider endpoint or workload" - -kubectl -n "$CANONICAL_SMOKE_NS" apply \ - -f config/samples/cachebackend-sglang.yaml >/dev/null \ - || fail "canonical Managed Redis CacheBackend sample failed to apply" - -deadline=$(($(date +%s) + CANONICAL_BACKEND_TIMEOUT)) -redis_endpoint="" -until kubectl -n "$CANONICAL_SMOKE_NS" get deployment "$CANONICAL_REDIS_CB" >/dev/null 2>&1 && \ - kubectl -n "$CANONICAL_SMOKE_NS" get service "$CANONICAL_REDIS_CB" >/dev/null 2>&1 && \ - [ -n "$redis_endpoint" ]; do - redis_endpoint="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_REDIS_CB" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$CANONICAL_SMOKE_NS" get cb,deploy,svc -o yaml || true - fail "canonical Managed Redis provider did not reconcile within ${CANONICAL_BACKEND_TIMEOUT}s" - fi - sleep 1 -done -redis_provider="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_REDIS_CB" \ - -o jsonpath='{.spec.remoteStorage.provider}' 2>/dev/null || true)" -redis_ownership="$(kubectl -n "$CANONICAL_SMOKE_NS" get cb "$CANONICAL_REDIS_CB" \ - -o jsonpath='{.spec.remoteStorage.ownership}' 2>/dev/null || true)" -redis_container="$(kubectl -n "$CANONICAL_SMOKE_NS" get deployment "$CANONICAL_REDIS_CB" \ - -o jsonpath='{.spec.template.spec.containers[0].name}' 2>/dev/null || true)" -redis_port="$(kubectl -n "$CANONICAL_SMOKE_NS" get service "$CANONICAL_REDIS_CB" \ - -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || true)" -expected_redis_endpoint="$CANONICAL_REDIS_CB.$CANONICAL_SMOKE_NS.svc.cluster.local:6379" -if [ "$redis_provider" != "Redis" ] || [ "$redis_ownership" != "Managed" ] || \ - [ "$redis_container" != "redis-l2" ] || [ "$redis_port" != "6379" ] || \ - [ "$redis_endpoint" != "$expected_redis_endpoint" ]; then - kubectl -n "$CANONICAL_SMOKE_NS" get cb,deploy,svc -o yaml || true - fail "canonical Managed Redis state is wrong: provider=$redis_provider ownership=$redis_ownership container=$redis_container port=$redis_port endpoint=$redis_endpoint" -fi -log "canonical Managed Redis hierarchy rendered redis-l2 and endpoint=$redis_endpoint" - -# Create (not only server-side dry-run) a matching SGLang Pod through the live -# webhook and inspect the object persisted by the apiserver. The impossible node -# selector keeps kubelet from pulling either large GPU image; admission still -# executes the complete mutation and Kubernetes schema/defaulting path. -kubectl -n "$CANONICAL_SMOKE_NS" apply \ - -f config/samples/cachebackend-sglang-podlocal-host-only.yaml >/dev/null \ - || fail "typed SGLang PodLocal CacheBackend sample failed to apply" - -typed_sglang_pod="$(mktemp "$tmpdir/sglang-podlocal-admission.XXXXXX.yaml")" -cat >"$typed_sglang_pod" <<'EOF' -apiVersion: v1 -kind: Pod -metadata: - name: sglang-podlocal-admission - labels: - inferencecache.io/runtime: sglang-mp -spec: - nodeSelector: - inferencecache.io/install-smoke-never-schedule: "true" - containers: - - name: sglang - image: example.invalid/sglang-lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa - command: ["python3", "-m", "sglang.launch_server"] - args: - - --model-path=meta-llama/Meta-Llama-3-8B-Instruct - - --page-size=64 - - --tensor-parallel-size=1 -EOF -kubectl -n "$CANONICAL_SMOKE_NS" create -f "$typed_sglang_pod" >/dev/null \ - || fail "matching typed SGLang Pod did not pass the live mutating webhook" - -typed_injected_by="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true)" -typed_metrics_label="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.metadata.labels.inferencecache\.io/lmcache-mp-metrics}' 2>/dev/null || true)" -typed_server_image="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].image}' 2>/dev/null || true)" -typed_server_restart="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].restartPolicy}' 2>/dev/null || true)" -typed_server_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].args}' 2>/dev/null || true)" -typed_server_memory_request="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].resources.requests.memory}' 2>/dev/null || true)" -typed_server_memory_limit="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].resources.limits.memory}' 2>/dev/null || true)" -typed_server_probe_path="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].readinessProbe.httpGet.path}' 2>/dev/null || true)" -typed_engine_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="sglang")].args}' 2>/dev/null || true)" -typed_engine_experimental="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="sglang")].env[?(@.name=="LMCACHE_USE_EXPERIMENTAL")].value}' 2>/dev/null || true)" -typed_engine_mounts="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="sglang")].volumeMounts[*].mountPath}' 2>/dev/null || true)" - -expected_mp_image="docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13" -if [ "$typed_injected_by" != "$CANONICAL_SMOKE_NS/$CANONICAL_TYPED_CB" ] || \ - [ "$typed_metrics_label" != "true" ] || \ - [ "$typed_server_image" != "$expected_mp_image" ] || \ - [ "$typed_server_restart" != "Always" ] || \ - [ "$typed_server_memory_request" != "5Gi" ] || \ - [ "$typed_server_memory_limit" != "6Gi" ] || \ - [ "$typed_server_probe_path" != "/healthcheck" ] || \ - [ "$typed_engine_experimental" != "True" ]; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true - fail "typed SGLang persisted wire metadata is wrong: injectedBy=$typed_injected_by metrics=$typed_metrics_label image=$typed_server_image restart=$typed_server_restart memory=$typed_server_memory_request/$typed_server_memory_limit probe=$typed_server_probe_path experimental=$typed_engine_experimental" -fi -if ! jq -e ' - (index("--port") as $port | $port != null and .[$port + 1] == "5555") and - (index("--chunk-size") as $chunk | $chunk != null and .[$chunk + 1] == "256") and - index("--l2-adapter") == null - ' >/dev/null <<<"$typed_server_args"; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true - fail "typed host-only SGLang MP server args are wrong: $typed_server_args" -fi -if ! jq -e ' - index("--page-size=64") != null and - index("--enable-lmcache") != null and - (index("--lmcache-config-file") as $config | $config != null and - .[$config + 1] == "/var/run/inference-cache/lmcache/client.yaml") - ' >/dev/null <<<"$typed_engine_args"; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true - fail "typed SGLang engine args are incomplete: $typed_engine_args" -fi -case " $typed_engine_mounts " in - *" /dev/shm "*) : ;; - *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true - fail "typed SGLang engine /dev/shm mount is missing: $typed_engine_mounts" ;; -esac -case " $typed_engine_mounts " in - *" /var/run/inference-cache/lmcache "*) : ;; - *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_POD" -o yaml || true - fail "typed SGLang engine config mount is missing: $typed_engine_mounts" ;; -esac -log "typed SGLang PodLocal Pod persisted with the common MP native sidecar and complete engine wire" - -kubectl -n "$CANONICAL_SMOKE_NS" apply \ - -f config/samples/cachebackend-vllm-podlocal-host-only.yaml >/dev/null \ - || fail "typed vLLM PodLocal CacheBackend sample failed to apply" - -typed_vllm_pod="$(mktemp "$tmpdir/vllm-podlocal-admission.XXXXXX.yaml")" -cat >"$typed_vllm_pod" <<'EOF' -apiVersion: v1 -kind: Pod -metadata: - name: vllm-podlocal-admission - labels: - inferencecache.io/runtime: vllm-mp -spec: - nodeSelector: - inferencecache.io/install-smoke-never-schedule: "true" - containers: - - name: vllm - image: example.invalid/vllm-lmcache@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb - command: ["python3", "-m", "vllm.entrypoints.openai.api_server"] - args: - - --model=meta-llama/Meta-Llama-3-8B-Instruct - - --tensor-parallel-size=2 -EOF -kubectl -n "$CANONICAL_SMOKE_NS" create -f "$typed_vllm_pod" >/dev/null \ - || fail "matching typed vLLM Pod did not pass the live mutating webhook" - -typed_vllm_injected_by="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.metadata.annotations.inferencecache\.io/injected-by}' 2>/dev/null || true)" -typed_vllm_metrics_label="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.metadata.labels.inferencecache\.io/lmcache-mp-metrics}' 2>/dev/null || true)" -typed_vllm_server_image="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].image}' 2>/dev/null || true)" -typed_vllm_server_restart="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-mp-server")].restartPolicy}' 2>/dev/null || true)" -typed_vllm_engine_args="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].args}' 2>/dev/null || true)" -typed_vllm_config="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o json \ - | jq -r '.spec.containers[] | select(.name == "vllm") | .args as $args | ($args | index("--kv-transfer-config")) as $index | if $index == null then "" else $args[$index + 1] end')" -typed_vllm_hash_seed="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="PYTHONHASHSEED")].value}' 2>/dev/null || true)" -typed_vllm_legacy_url="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="LMCACHE_REMOTE_URL")].value}' 2>/dev/null || true)" -typed_vllm_engine_mounts="$(kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].volumeMounts[*].mountPath}' 2>/dev/null || true)" - -if [ "$typed_vllm_injected_by" != "$CANONICAL_SMOKE_NS/$CANONICAL_TYPED_VLLM_CB" ] || \ - [ "$typed_vllm_metrics_label" != "true" ] || \ - [ "$typed_vllm_server_image" != "$expected_mp_image" ] || \ - [ "$typed_vllm_server_restart" != "Always" ] || \ - [ "$typed_vllm_hash_seed" != "0" ] || \ - [ -n "$typed_vllm_legacy_url" ]; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true - fail "typed vLLM persisted wire metadata is wrong: injectedBy=$typed_vllm_injected_by metrics=$typed_vllm_metrics_label image=$typed_vllm_server_image restart=$typed_vllm_server_restart hashSeed=$typed_vllm_hash_seed legacyURL=$typed_vllm_legacy_url" -fi -if ! jq -e ' - .kv_connector == "LMCacheMPConnector" and - .kv_connector_module_path == "lmcache.integration.vllm.lmcache_mp_connector" and - .kv_role == "kv_both" and - .kv_connector_extra_config["lmcache.mp.host"] == "tcp://127.0.0.1" and - .kv_connector_extra_config["lmcache.mp.port"] == "5555" - ' >/dev/null <<<"$typed_vllm_config"; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true - fail "typed vLLM connector JSON is wrong: $typed_vllm_config" -fi -if ! jq -e ' - index("--tensor-parallel-size=2") != null and - index("--disable-hybrid-kv-cache-manager") != null and - index("--kv-transfer-config") != null - ' >/dev/null <<<"$typed_vllm_engine_args"; then - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true - fail "typed vLLM engine args are incomplete: $typed_vllm_engine_args" -fi -case " $typed_vllm_engine_mounts " in - *" /dev/shm "*) : ;; - *) kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true - fail "typed vLLM engine /dev/shm mount is missing: $typed_vllm_engine_mounts" ;; -esac -case " $typed_vllm_engine_mounts " in - *" /var/run/inference-cache/lmcache "*) - kubectl -n "$CANONICAL_SMOKE_NS" get pod "$CANONICAL_TYPED_VLLM_POD" -o yaml || true - fail "typed vLLM unexpectedly received the SGLang YAML config mount: $typed_vllm_engine_mounts" ;; -esac -log "typed vLLM PodLocal Pod persisted with the dedicated external MP connector wire" - -kubectl delete namespace "$CANONICAL_SMOKE_NS" \ - --wait=false --ignore-not-found=true >/dev/null 2>&1 || true - -# --- legacy IP External compatibility -------------------------------------- -# INTENTIONAL LEGACY FIXTURE: this section exercises External LMCacheServer/IP -# validation and injection retained until Phase 7. It is not a production -# reference. Current External Redis is covered by typed schema/sample checks. -# -# Exercises External passthrough on the running cluster: -# the mutating webhook should stamp spec.replicas, the reconciler should NOT -# render a Deployment/Service, status.endpoint should mirror -# spec.remoteStorage.endpoint, -# observedGeneration should advance, Ready should be True with reason -# ExternalEndpointAccepted, and a matching engine pod should come out of -# admission with LMCACHE_REMOTE_URL pointing at the operator-supplied -# endpoint. Also exercises CacheBackend printer columns and the validating -# webhook's negative path. -log "exercising External CacheBackend end-to-end in namespace $EXT_SMOKE_NS" -kubectl create namespace "$EXT_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -# The inline fixture intentionally omits spec.replicas so the smoke drives the -# mutating webhook defaulter instead of only proving the CRD accepts already- -# defaulted YAML. -cat </dev/null \ - || fail "kubectl apply legacy External LMCacheServer fixture failed" -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: $EXT_SMOKE_CB_NAME -spec: - runtime: VLLM - type: LMCache - integration: - role: ReadWrite - engineSelector: - matchLabels: - app.kubernetes.io/name: vllm - remoteStorage: - provider: LMCacheServer - ownership: External - endpoint: my-cache.example.com:8200 -EOF - -defaulted_replicas="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" -external_spec_endpoint="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.spec.remoteStorage.endpoint}' 2>/dev/null || true)" -external_pod_labels="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o go-template='{{range $k, $v := .spec.engineSelector.matchLabels}}{{printf " %s: %s\n" $k $v}}{{end}}' \ - 2>/dev/null || true)" -if [ -z "$external_spec_endpoint" ]; then - kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" -o yaml || true - fail "External sample did not create spec.remoteStorage.endpoint on $EXT_SMOKE_CB_NAME" -fi -if [ -z "$external_pod_labels" ]; then - kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" -o yaml || true - fail "External sample did not create spec.engineSelector.matchLabels on $EXT_SMOKE_CB_NAME" -fi -if [ "$defaulted_replicas" != "1" ]; then - kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" -o yaml || true - fail "CacheBackend defaulter did not stamp spec.replicas: got=$defaulted_replicas (want 1)" -fi -log "External CR sample endpoint=$external_spec_endpoint; defaulted replicas=$defaulted_replicas" - -# Wait for the reconciler to publish status.endpoint + observedGeneration + -# the Ready=True condition. Sub-second on a quiet cluster; the timeout covers -# leader-election warm-up. -log "waiting up to ${EXTERNAL_BACKEND_TIMEOUT}s for External CR to publish status + Ready=True" -deadline=$(($(date +%s) + EXTERNAL_BACKEND_TIMEOUT)) -status_endpoint="" -observed_generation="" -metadata_generation="" -ready_status="" -ready_reason="" -until [ "$status_endpoint" = "$external_spec_endpoint" ] && \ - [ -n "$observed_generation" ] && \ - [ "$ready_status" = "True" ] && \ - [ "$ready_reason" = "ExternalEndpointAccepted" ]; do - status_endpoint="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" - observed_generation="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true)" - metadata_generation="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.metadata.generation}' 2>/dev/null || true)" - ready_status="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - ready_reason="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" -o yaml || true - fail "External CR didn't converge: status.endpoint=$status_endpoint observedGeneration=$observed_generation Ready=$ready_status/$ready_reason (want $external_spec_endpoint non-empty True/ExternalEndpointAccepted)" - fi - sleep 1 -done -if [ "$observed_generation" != "$metadata_generation" ]; then - kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" -o yaml || true - fail "External CR status.observedGeneration=$observed_generation, want metadata.generation=$metadata_generation" -fi -log "External CR status: endpoint=$status_endpoint observedGeneration=$observed_generation Ready=$ready_status/$ready_reason" - -# The CacheBackend printer columns are an operator-facing surface. Verify the -# table exposes the expected columns and row values instead of regressing to a -# default NAME/AGE-only table. HEALTH was retired in favour of the Ready -# condition printer column (see this PR's design-doc carve-out); the row -# value is the condition's True string rather than the old enum value. -cb_table="$(kubectl -n "$EXT_SMOKE_NS" get cb "$EXT_SMOKE_CB_NAME" 2>/dev/null || true)" -cb_header="$(printf '%s\n' "$cb_table" | sed -n '1p')" -for column in TYPE READY MATCHED ENDPOINT PREFIXES LASTEVENT; do - if ! grep -Eq "(^|[[:space:]])${column}([[:space:]]|$)" <<<"$cb_header"; then - echo "$cb_table" - fail "expected CacheBackend printer column $column in kubectl get cb output" - fi -done -if ! grep -Fq "$EXT_SMOKE_CB_NAME" <<<"$cb_table" || \ - ! grep -Fq "LMCache" <<<"$cb_table" || \ - ! grep -Fq "True" <<<"$cb_table" || \ - ! grep -Fq "$external_spec_endpoint" <<<"$cb_table"; then - echo "$cb_table" - fail "expected CacheBackend printer row to include name/type=LMCache/Ready=True/endpoint" -fi -log "CacheBackend printer columns render Type/Ready/Matched/Endpoint/Prefixes/LastEvent" - -# No Deployment, no Service should have been rendered for an External CR. -# A leading API service `kubernetes` doesn't exist in this fresh namespace, -# so a flat count of zero is the right assertion. -dep_count="$(kubectl -n "$EXT_SMOKE_NS" get deploy -o name 2>/dev/null | wc -l | tr -d ' ')" -svc_count="$(kubectl -n "$EXT_SMOKE_NS" get svc -o name 2>/dev/null | wc -l | tr -d ' ')" -if [ "$dep_count" != "0" ] || [ "$svc_count" != "0" ]; then - kubectl -n "$EXT_SMOKE_NS" get deploy,svc - fail "External CR rendered controller-owned workload (deploy=$dep_count svc=$svc_count, want 0/0)" -fi -log "no Deployment or Service in $EXT_SMOKE_NS (External backend skipped provisioning)" - -# Apply a matching engine pod with the conventional `vllm` container name. -# `pause` keeps the pod alive long enough to inspect the injected env+args -# without pulling a real vLLM image (which would be ~5+ GB). -cat </dev/null || fail "kubectl apply engine pod failed" -apiVersion: v1 -kind: Pod -metadata: - name: $EXT_SMOKE_POD_NAME - namespace: $EXT_SMOKE_NS - labels: -$external_pod_labels -spec: - containers: - - name: vllm - image: registry.k8s.io/pause:3.10 -EOF - -# The pod webhook is synchronous at admission, so the env should be present -# the moment the API has the object. The retry loop here is a defensive -# circuit-breaker against a slow first-admission (cert-manager certificate -# becoming available, etc.), NOT an expected wait. -log "waiting up to ${EXTERNAL_INJECT_TIMEOUT}s for pod webhook to inject External endpoint" -deadline=$(($(date +%s) + EXTERNAL_INJECT_TIMEOUT)) -injected="" -until [ -n "$injected" ]; do - injected="$(kubectl -n "$EXT_SMOKE_NS" get pod "$EXT_SMOKE_POD_NAME" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="LMCACHE_REMOTE_URL")].value}' \ - 2>/dev/null || true)" - if [ -n "$injected" ]; then break; fi - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$EXT_SMOKE_NS" get pod "$EXT_SMOKE_POD_NAME" -o yaml || true - fail "pod webhook did not inject LMCACHE_REMOTE_URL within ${EXTERNAL_INJECT_TIMEOUT}s" - fi - sleep 1 -done -# Form the expected URL the same way the adapter does: preserve an -# operator-supplied `lm://` prefix (case-insensitive — admission lowers -# the scheme), otherwise prepend it. Without this a sample endpoint of -# `lm://host:port` (legal per the contract) would compare against -# `lm://lm://host:port` and the smoke would fail on a valid input. -case "$(printf '%s' "$external_spec_endpoint" | tr '[:upper:]' '[:lower:]')" in - lm://*) expected_url="lm://${external_spec_endpoint#??://}" ;; - *) expected_url="lm://$external_spec_endpoint" ;; -esac -if [ "$injected" != "$expected_url" ]; then - fail "LMCACHE_REMOTE_URL=$injected, want $expected_url (pod webhook should wire to spec.remoteStorage.endpoint via the LMCache wire format)" -fi -log "pod webhook injected LMCACHE_REMOTE_URL=$injected" - -# Host networking is Mooncake's carve-out and must not leak past it. This pod is -# a genuinely admitted, genuinely injected NON-Mooncake engine pod (the assertion -# above proves the webhook ran on it), so an empty hostNetwork here means the -# webhook left it alone — not that the check silently skipped. A regression that -# moved this onto the host network would break every Pod Security "restricted" -# namespace running the shipping default. -ext_pod_hostnet="$(kubectl -n "$EXT_SMOKE_NS" get pod "$EXT_SMOKE_POD_NAME" \ - -o jsonpath='{.spec.hostNetwork}' 2>/dev/null || true)" -if [ -n "$ext_pod_hostnet" ] && [ "$ext_pod_hostnet" != "false" ]; then - kubectl -n "$EXT_SMOKE_NS" get pod "$EXT_SMOKE_POD_NAME" -o yaml || true - fail "non-Mooncake engine pod hostNetwork=$ext_pod_hostnet, want unset/false (the host-network carve-out must stay Mooncake-scoped)" -fi -log "non-Mooncake engine pod stays on the pod network (hostNetwork carve-out is Mooncake-scoped)" - -# Verify the --kv-transfer-config arg is also present — pins the full -# engine wire contract canonical External ownership uses through the LMCache -# runtime adapter. -kv_arg="$(kubectl -n "$EXT_SMOKE_NS" get pod "$EXT_SMOKE_POD_NAME" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].args}' 2>/dev/null || true)" -if ! grep -q -- "--kv-transfer-config" <<<"$kv_arg"; then - fail "pod webhook did not inject --kv-transfer-config arg; got args=$kv_arg" -fi -log "pod args contain --kv-transfer-config" - -# Negative-path admission checks. Each must be rejected with a specific -# message; admission error goes to stderr so we capture both streams. -log "exercising negative admission rules" - -reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-no-endpoint - namespace: $EXT_SMOKE_NS -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: External -EOF -)" -if ! grep -q "required when remoteStorage.ownership=External" <<<"$reject_output"; then - fail "admission did not reject external ownership with no endpoint as expected; got: $reject_output" -fi - -reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-https - namespace: $EXT_SMOKE_NS -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: External - endpoint: https://cache.example.com:443/api -EOF -)" -if ! grep -q 'scheme "https" is not supported' <<<"$reject_output"; then - fail "admission did not reject external LMCacheServer+https scheme as expected; got: $reject_output" -fi - -reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-no-host - namespace: $EXT_SMOKE_NS -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: External - endpoint: "lm://" -EOF -)" -if ! grep -q "must be a non-empty host AND port" <<<"$reject_output"; then - fail "admission did not reject external LMCacheServer+lm:// (no host) as expected; got: $reject_output" -fi - -reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-managed-endpoint - namespace: $EXT_SMOKE_NS -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: Managed - endpoint: user-supplied.example:8080 -EOF -)" -if ! grep -q "managed providers publish their observed endpoint" <<<"$reject_output"; then - fail "admission did not reject managed remote storage + endpoint as expected; got: $reject_output" -fi - -# Canonical runtime/cache adapters must explicitly accept their remote binding. -# Native SGLang HiCache is engine-local and accepts only a nil binding, so a -# Redis provider must be rejected before the controller could provision an -# unused remote tier. -reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-hicache-redis - namespace: $EXT_SMOKE_NS -spec: - runtime: SGLang - type: SGLangHiCache - hiCache: - ratio: "2" - engineSelector: - matchLabels: - app: sglang-hicache - remoteStorage: - provider: Redis - ownership: Managed - redis: {} -EOF -)" -if ! grep -q 'does not accept remote-storage protocol "resp"' <<<"$reject_output"; then - fail "admission did not reject canonical SGLangHiCache + Redis as expected; got: $reject_output" -fi -log "admission rejected invalid canonical endpoint shapes and SGLangHiCache + Redis" - -# --- CacheBackend admission: scale-to-zero + autoscaling + nil minReplicas --- -# The installed validating webhook must reject the combination -# spec.replicas=0 + spec.autoscaling enabled + nil spec.autoscaling.minReplicas. -# With replicas=0 the defaulter declines to stamp minReplicas (a 0 value would -# violate the schema's Minimum=1), so without this rule the apiserver would -# accept the CR with minReplicas unset and the reconciler's HPA fallback would -# silently pick minReplicas=1 — turning an operator's "scale to zero" into -# "scale 1-N" with no notification. This is an operator-facing surface (an -# operator hits it on `kubectl apply` of a misconfigured CR), so it needs a -# real-install smoke assertion, not just the envtest/unit coverage. -# -# Uses a dedicated INTENTIONALLY-INVALID fixture parked under -# config/samples/_test/ with a `# verify-samples: skip` opt-out so neither -# `make verify-samples` nor the apply-clean backstop (final phase) treats it as -# a shippable sample. The fixture omits a namespace, so it is applied into -# $EXT_SMOKE_NS (still present here). -scale_to_zero_fixture="config/samples/_test/cachebackend-invalid-scale-to-zero-no-min.yaml" -scale_to_zero_name="cachebackend-invalid-scale-to-zero-no-min" -log "asserting a replicas=0 + autoscaling + nil-minReplicas CacheBackend is rejected at admission" -if scale_to_zero_out="$(kubectl apply -n "$EXT_SMOKE_NS" -f "$scale_to_zero_fixture" 2>&1)"; then - echo "$scale_to_zero_out" - fail "replicas=0 + autoscaling + nil minReplicas CacheBackend was admitted; the scale-to-zero-requires-explicit-minReplicas rule did not fire on the real install" -fi -# Match wording-stable substrings so a minor rewording of the message doesn't -# break the gate, while still proving the rejection came from THIS rule and not -# some unrelated admission failure (which would false-pass a bare exit-code -# check). All three tokens are load-bearing phrases of the locked message. -for token in "spec.replicas=0" "minReplicas" "scale to zero"; do - if ! grep -qF "$token" <<<"$scale_to_zero_out"; then - echo "$scale_to_zero_out" - fail "scale-to-zero CacheBackend was rejected, but not by the expected scale-to-zero-requires-explicit-minReplicas rule (missing substring '$token' in the admission message)" - fi -done -# The rejection must have prevented persistence — a rejected CREATE writes -# nothing, so the object must be absent. `kubectl get` on a missing object exits -# non-zero with "NotFound"; anything else (the CR present, or a different error) -# means the reject-then-not-persisted contract broke. -if get_out="$(kubectl get cachebackend -n "$EXT_SMOKE_NS" "$scale_to_zero_name" 2>&1)"; then - echo "$get_out" - fail "rejected scale-to-zero CacheBackend '$scale_to_zero_name' was persisted in $EXT_SMOKE_NS; admission reject must not create the object" -fi -if ! grep -q "NotFound\|not found" <<<"$get_out"; then - echo "$get_out" - fail "could not confirm scale-to-zero CacheBackend '$scale_to_zero_name' is absent (expected a NotFound, got an unexpected kubectl error): $get_out" -fi -log "replicas=0 + autoscaling + nil-minReplicas CacheBackend rejected at admission and not persisted" - -# Clean up — keeps the cluster reusable for KEEP_CLUSTER=1 reruns. -kubectl delete pod -n "$EXT_SMOKE_NS" "$EXT_SMOKE_POD_NAME" --ignore-not-found --wait=false >/dev/null || true -kubectl delete cb -n "$EXT_SMOKE_NS" "$EXT_SMOKE_CB_NAME" --ignore-not-found --wait=false >/dev/null || true -kubectl delete namespace "$EXT_SMOKE_NS" --ignore-not-found --wait=false >/dev/null || true - -# --- Events-only CacheBackend end-to-end ----------------------------------- -# Exercises spec.integration.mode=EventsOnly (the routing-only integration) on -# the running cluster. The operator-facing contract for an events-only backend -# is the inverse of a managed one: the reconciler provisions NO owned Deployment -# and NO owned Service, status.endpoint stays EMPTY (no server address to -# publish), and readiness runs the same KV-event gate as a managed backend. -# -# The default install wires no --kvevent-subscriber-image, so no subscriber -# sidecar is injected and no KV events ever flow. The events-only backend is -# server-less, so it is "up" the moment it exists (status.firstAvailableAt -# latches immediately) and the gate parks it at Ready=False/AwaitingFirstKVEvent -# inside the firstEventTimeout window — exactly the managed sample's -# no-KV-event-source assertion above, but with no Deployment to wait on. The -# managed-only advisory conditions (FunctionalProbeOK / EngineKernelsHealthy / -# T2Degraded / EngineCompatibility) must be ABSENT — events-only has no server -# to probe, loads no LMCache connector whose native kernels need checking, has -# no tier-2, and injects no connector that could be incompatible. -# We assert the server-less + empty-endpoint + KV-gate contract rather than a -# positive KV event, since the smoke's engine stand-in emits none. -log "exercising Events-only CacheBackend end-to-end in namespace $EVENTSONLY_SMOKE_NS" -kubectl create namespace "$EVENTSONLY_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -# Apply the COMMITTED events-only sample so it is exercised end-to-end and -# cannot silently drift — every other backend type's smoke phase applies its -# config/samples/ manifest (with-engine, external, cachepolicy, cachetenant), -# so this one must too rather than hand-typing a private inline copy. The -# sample's metadata.name is cachebackend-events-only == the default -# $EVENTSONLY_SMOKE_CB_NAME; pin the name via a tmp copy so an overridden -# tunable still resolves, and set the namespace with -n (the sample is -# namespace-less, like the other samples). type=LMCache, integration.mode= -# EventsOnly, observation.modelID set, no remoteStorage, and no spec.autoscaling -# (rejected for events-only). The -# sample's engineSelector is irrelevant here: events-only provisions no -# workload and the assertions below are all about the CR's own reconcile, so -# no matched engine pod is required. -eo_sample_tmp="$(mktemp "$tmpdir/sample-events-only.XXXXXX")" -sed "s|^ name: cachebackend-events-only\$| name: $EVENTSONLY_SMOKE_CB_NAME|" \ - config/samples/cachebackend-events-only.yaml > "$eo_sample_tmp" -kubectl -n "$EVENTSONLY_SMOKE_NS" apply -f "$eo_sample_tmp" >/dev/null \ - || fail "kubectl apply events-only sample (config/samples/cachebackend-events-only.yaml) failed" - -# Wait for the reconciler to take the events-only path: Ready published by the -# KV-event gate (False/AwaitingFirstKVEvent before any event), status.endpoint -# empty, observedGeneration advanced. -log "waiting up to ${EVENTSONLY_BACKEND_TIMEOUT}s for events-only CR to reach Ready=False/AwaitingFirstKVEvent" -deadline=$(($(date +%s) + EVENTSONLY_BACKEND_TIMEOUT)) -eo_ready_status="" -eo_ready_reason="" -eo_observed_generation="" -until [ "$eo_ready_status" = "False" ] && \ - [ "$eo_ready_reason" = "AwaitingFirstKVEvent" ] && \ - [ -n "$eo_observed_generation" ]; do - eo_ready_status="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - eo_ready_reason="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].reason}' 2>/dev/null || true)" - eo_observed_generation="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true)" - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" -o yaml || true - fail "events-only CR didn't converge: Ready=$eo_ready_status/$eo_ready_reason observedGeneration=$eo_observed_generation (want False/AwaitingFirstKVEvent + advanced generation)" - fi - sleep 1 -done -log "events-only CR Ready=$eo_ready_status/$eo_ready_reason observedGeneration=$eo_observed_generation" - -# status.endpoint must stay EMPTY — events-only provisions no server, so there -# is no address to publish. (A managed/External backend mirrors one here.) -eo_endpoint="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" -if [ -n "$eo_endpoint" ]; then - kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" -o yaml || true - fail "events-only status.endpoint=$eo_endpoint, want empty (no provisioned server)" -fi - -# No owned Deployment, no owned Service. The namespace is dedicated to this -# phase and otherwise empty, so a flat count of zero is the right assertion -# (matches the External phase's reasoning). -eo_dep_count="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get deploy -o name 2>/dev/null | wc -l | tr -d ' ')" -eo_svc_count="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get svc -o name 2>/dev/null | wc -l | tr -d ' ')" -if [ "$eo_dep_count" != "0" ] || [ "$eo_svc_count" != "0" ]; then - kubectl -n "$EVENTSONLY_SMOKE_NS" get deploy,svc - fail "events-only CR rendered controller-owned workload (deploy=$eo_dep_count svc=$eo_svc_count, want 0/0)" -fi -log "no Deployment or Service in $EVENTSONLY_SMOKE_NS (events-only backend skipped provisioning)" - -# The KV-event gate latch must be unset: no KV event source exists (no -# subscriber image wired), so the controller must never write -# status.firstKVEventObservedAt — same invariant as the managed gate phase. -eo_latch="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath='{.status.firstKVEventObservedAt}' 2>/dev/null || true)" -if [ -n "$eo_latch" ]; then - kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" -o yaml || true - fail "events-only status.firstKVEventObservedAt=$eo_latch, want unset (no KV event source exists)" -fi - -# The managed-only advisory conditions must be ABSENT on an events-only backend: -# events-only publishes only Ready/Degraded/Progressing. EngineCompatibility is -# included because events-only injects no connector (nothing to be incompatible -# with) and an Offload->EventsOnly flip clears any prior verdict — this asserts -# that clear holds at install level, not just in envtest. -for eo_cond in FunctionalProbeOK EngineKernelsHealthy T2Degraded EngineCompatibility; do - eo_present="$(kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" \ - -o jsonpath="{.status.conditions[?(@.type=='$eo_cond')].type}" 2>/dev/null || true)" - if [ -n "$eo_present" ]; then - kubectl -n "$EVENTSONLY_SMOKE_NS" get cb "$EVENTSONLY_SMOKE_CB_NAME" -o yaml || true - fail "events-only CR published managed-only condition $eo_cond, want absent" - fi -done -log "events-only CR publishes only Ready/Degraded/Progressing (FunctionalProbeOK/EngineKernelsHealthy/T2Degraded/EngineCompatibility absent)" - -# Negative-path admission: the misconfiguration the events-only validator -# guards. EventsOnly plus externally owned remote storage must be rejected at -# admission because events-only wires no connector that could dial it. -eo_reject_output="$(kubectl apply -f - <&1 || true -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: smoke-reject-events-only-external - namespace: $EVENTSONLY_SMOKE_NS -spec: - runtime: VLLM - type: LMCache - remoteStorage: - provider: LMCacheServer - ownership: External - endpoint: external-cache.example:8200 - integration: - mode: EventsOnly -EOF -)" -if ! grep -q "provision no remote-storage provider" <<<"$eo_reject_output"; then - fail "admission did not reject EventsOnly+external remote storage as expected; got: $eo_reject_output" -fi -log "admission rejected EventsOnly+external remote-storage misconfiguration" - -# Clean up — keeps the cluster reusable for KEEP_CLUSTER=1 reruns. -kubectl delete cb -n "$EVENTSONLY_SMOKE_NS" "$EVENTSONLY_SMOKE_CB_NAME" --ignore-not-found --wait=false >/dev/null || true -kubectl delete namespace "$EVENTSONLY_SMOKE_NS" --ignore-not-found --wait=false >/dev/null || true - -# --- Native SGLang HiCache end-to-end -------------------------------------- -# This phase drives the new operator-facing surface through the real installed -# CRD, validating webhook, controller, and Pod mutating webhook. HiCache is -# engine-local, so the observable controller contract is deliberately negative: -# no cache-server workload, no endpoint, and no synthetic Ready condition. The -# Pod is server-side dry-run only; its admitted shape proves native argument -# injection without pulling or starting a real SGLang image. -log "exercising native SGLang HiCache end-to-end in namespace $HICACHE_SMOKE_NS" -kubectl create namespace "$HICACHE_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -# Apply the committed sample rather than duplicating its CacheBackend contract -# inline. Only the name is tunable; the sample remains namespace-less so -n -# places it in the dedicated smoke namespace. -hc_sample_tmp="$(mktemp "$tmpdir/sample-sglang-hicache.XXXXXX")" -sed "s|^ name: sglang-hicache\$| name: $HICACHE_SMOKE_CB_NAME|" \ - config/samples/cachebackend-sglang-hicache.yaml > "$hc_sample_tmp" -kubectl -n "$HICACHE_SMOKE_NS" apply -f "$hc_sample_tmp" >/dev/null \ - || fail "kubectl apply native SGLang HiCache sample failed" - -# Wait until the controller has taken the engine-local path. observedGeneration -# is the positive acknowledgement; endpoint/Ready/workload assertions below pin -# what that path intentionally does not publish or provision. -hc_generation="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ - -o jsonpath='{.metadata.generation}')" -deadline=$(($(date +%s) + HICACHE_SMOKE_TIMEOUT)) -hc_observed_generation="" -until [ "$hc_observed_generation" = "$hc_generation" ]; do - hc_observed_generation="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ - -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true)" - if [ "$(date +%s)" -ge "$deadline" ]; then - kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" -o yaml || true - fail "native HiCache CR was not observed within ${HICACHE_SMOKE_TIMEOUT}s: generation=$hc_generation observedGeneration=$hc_observed_generation" - fi - sleep 1 -done -log "native HiCache CR observed at generation $hc_observed_generation" - -hc_endpoint="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true)" -hc_ready="$(kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" \ - -o jsonpath='{.status.conditions[?(@.type=="Ready")].type}' 2>/dev/null || true)" -if [ -n "$hc_endpoint" ] || [ -n "$hc_ready" ]; then - kubectl -n "$HICACHE_SMOKE_NS" get cb "$HICACHE_SMOKE_CB_NAME" -o yaml || true - fail "native HiCache published server-backed status (endpoint=$hc_endpoint Ready=$hc_ready, want both absent)" -fi - -hc_dep_count="$(kubectl -n "$HICACHE_SMOKE_NS" get deploy -o name 2>/dev/null | wc -l | tr -d ' ')" -hc_svc_count="$(kubectl -n "$HICACHE_SMOKE_NS" get svc -o name 2>/dev/null | wc -l | tr -d ' ')" -hc_hpa_count="$(kubectl -n "$HICACHE_SMOKE_NS" get hpa -o name 2>/dev/null | wc -l | tr -d ' ')" -if [ "$hc_dep_count" != "0" ] || [ "$hc_svc_count" != "0" ] || [ "$hc_hpa_count" != "0" ]; then - kubectl -n "$HICACHE_SMOKE_NS" get deploy,svc,hpa || true - fail "native HiCache rendered controller-owned workload (deploy=$hc_dep_count svc=$hc_svc_count hpa=$hc_hpa_count, want 0/0/0)" -fi -log "native HiCache rendered no Deployment, Service, or HPA and published no endpoint/Ready condition" - -# Exercise the installed Pod mutating webhook with a matching, single-container -# engine Pod. The dry-run response is the fully admitted Pod, including webhook -# mutations, but nothing is persisted or started. -hc_engine_fixture="$(mktemp "$tmpdir/pod-sglang-hicache.XXXXXX.yaml")" -cat > "$hc_engine_fixture" </dev/null)"; then - fail "matching SGLang Pod did not pass server-side dry-run admission" -fi -hc_expected_args=$'sleep\n3600\n--enable-hierarchical-cache\n--hicache-ratio\n2.0\n--hicache-write-policy\nwrite_through\n--hicache-io-backend\nkernel\n--hicache-mem-layout\nlayer_first\n--enable-metrics' -if [ "$hc_pod_args" != "$hc_expected_args" ]; then - printf '[install-smoke] admitted SGLang args:\n%s\n' "$hc_pod_args" >&2 - fail "native HiCache Pod mutation did not produce the expected complete argument contract" -fi - -hc_injected_by="$(kubectl create --dry-run=server --request-timeout=30s \ - -f "$hc_engine_fixture" \ - -o go-template='{{index .metadata.annotations "inferencecache.io/injected-by"}}' 2>/dev/null)" \ - || fail "could not read native HiCache injection annotation from dry-run Pod" -if [ "$hc_injected_by" != "$HICACHE_SMOKE_NS/$HICACHE_SMOKE_CB_NAME" ]; then - fail "native HiCache dry-run Pod injected-by=$hc_injected_by, want $HICACHE_SMOKE_NS/$HICACHE_SMOKE_CB_NAME" -fi -log "native HiCache Pod webhook injected the complete CLI contract and backend identity" - -kubectl delete cb -n "$HICACHE_SMOKE_NS" "$HICACHE_SMOKE_CB_NAME" --ignore-not-found --wait=false >/dev/null || true -kubectl delete namespace "$HICACHE_SMOKE_NS" --ignore-not-found --wait=false >/dev/null || true - -# --- /snapshot NetworkPolicy-drop assertion ------------------------------- -# The CacheIndex CR being populated above already proves the controller can -# scrape /snapshot with its SA token (the bearer path). The complementary -# half — that an UNAUTHENTICATED caller is dropped BY THE NETWORKPOLICY — is -# what this section checks. A short-lived curl pod outside the controller's -# identity (and outside the NetworkPolicy allowlist: it carries none of the -# component=controller labels) tries to GET /snapshot. Because the cluster runs -# a NetworkPolicy-enforcing CNI (Calico — see install_calico), the server -# NetworkPolicy drops the SYN at L3/L4, so curl times out (`-m 5`) and exits 28 -# WITHOUT ever reaching the L7 auth middleware. That drop is the required -# outcome. A bare HTTP 401 is NOT accepted here: a 401 means the probe -# reached the listener, i.e. the NetworkPolicy was deleted/broken and only the -# auth middleware is still standing — exactly the regression this gate must -# catch. (The auth-middleware path is still exercised, positively, by the -# audience-binding assertion further below, whose probe pod IS in the allowlist.) -log "asserting unauthenticated /snapshot scrape from a side pod is dropped by the NetworkPolicy" -SIDE_POD="ic-snapshot-probe" -# Clean any leftover probe from an interrupted prior run before creating a -# fresh one — otherwise `kubectl run` fails with AlreadyExists and the script -# silently reads stale logs from the previous attempt. --wait gates on the -# delete actually completing so the create below sees a clean namespace. -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD" --ignore-not-found --wait=true >/dev/null 2>&1 || true -if ! kubectl -n "$NAMESPACE" run "$SIDE_POD" --image=curlimages/curl:8.10.1 --restart=Never \ - --command -- /bin/sh -c ' - # -w prints the HTTP status; -o /dev/null discards the (error) body so the - # status is the only line on stdout. Timeout protects against the listener - # being unreachable (NetworkPolicy drop), in which case curl exits non-zero. - curl -sS -m 5 -o /dev/null -w "%{http_code}" \ - http://inference-cache-server:8081/snapshot || echo "curl_failed:$?" - ' >/tmp/snapshot-probe-create.log 2>&1; then - cat /tmp/snapshot-probe-create.log >&2 || true - fail "kubectl run $SIDE_POD failed; cannot run /snapshot auth assertion" -fi - -# Wait for the probe pod to finish (Succeeded or Failed) — either is fine; we -# read its logs to learn the outcome. The 90s budget covers the curlimages/curl -# image pull on a cold kind node (typical pull is ~15s, but the paired-sample -# phase that runs earlier can leave the kubelet busy reaping its own -# Terminating pods, occasionally pushing the new pod's container creation -# above the previous 30s budget). -for _ in $(seq 1 90); do - phase="$(kubectl -n "$NAMESPACE" get pod "$SIDE_POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ]; then - break - fi - sleep 1 -done -probe_out="$(kubectl -n "$NAMESPACE" logs "$SIDE_POD" 2>/dev/null || true)" -# If the pod never finished, capture its describe output so the failure -# message tells operators why (ImagePullBackOff vs ContainerCreating vs ...). -if [ -z "$probe_out" ]; then - kubectl -n "$NAMESPACE" describe pod "$SIDE_POD" >&2 || true -fi -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD" --grace-period=0 --force >/dev/null 2>&1 || true - -# Required outcome (the NetworkPolicy must do the work): -# - "curl_failed:28": curl timed out (`-m 5`), i.e. the L3/L4 NetworkPolicy -# dropped the SYN and the kernel never saw a RST. Exit code 28 is the -# SHAPE of a real CNI-enforced policy drop, and under the enforcing CNI it -# is the ONLY outcome an out-of-allowlist pod can get. -# Rejected outcomes: -# - 401 (auth middleware) → the probe REACHED the listener, so the L3/L4 -# NetworkPolicy did not drop it. That means the server NetworkPolicy is -# missing/broken or the CNI stopped enforcing — the regression this gate -# exists to catch. Previously accepted as a kindnet fallback; no longer. -# - 200 (unauthenticated read) → always a regression (both gates down). -# - 6 (couldn't resolve host) → Service rename or DNS bug. -# - 7 (failed to connect, e.g. ECONNREFUSED) → server not listening; an -# enforcing CNI drops packets silently, it does not RST, so 7 would -# mask a "listener crashed" bug. -# - 3 (malformed URL) → script regression. -case "$probe_out" in - *"curl_failed:28"*) - log "unauthenticated /snapshot probe dropped by the NetworkPolicy (curl timed out; probe output: $probe_out)" - ;; - "401") - fail "unauthenticated /snapshot probe reached the listener and got HTTP 401 — the L7 auth middleware answered, but the server NetworkPolicy did NOT drop the connection at L3/L4 (expected curl_failed:28). The NetworkPolicy is missing/broken or the CNI is not enforcing it. Got: $probe_out" - ;; - *) - fail "unauthenticated /snapshot probe was not dropped by the NetworkPolicy (expected curl_failed:28); got: $probe_out" - ;; -esac - -# --- /policy NetworkPolicy-drop assertion ----------------------------------- -# The CachePolicy side-effect assertion above proves the authenticated write -# path works: the controller pushed the CR through /policy and the server -# enforced it on LookupRoute. The complementary half — that an UNAUTHENTICATED -# POST is DROPPED BY THE NETWORKPOLICY — is what this section checks, since -# /policy is replace-on-write and a successful rogue POST would override -# cluster-wide policy state with no audit trail. Mirror of the /snapshot probe -# above: the side pod carries none of the controller labels, so under the -# enforcing CNI its connection to :8081 is dropped and curl exits 28 before the -# body is ever sent. The body is still a valid PolicySnapshot so that if the -# NetworkPolicy IS broken the probe gets a clean 401 (auth) rather than a 400 -# (bad request) — making the failure diagnosable. Required outcome is -# curl_failed:28; a bare 401 now FAILS. -log "asserting unauthenticated /policy POST from a side pod is dropped by the NetworkPolicy" -SIDE_POD_POLICY="ic-policy-probe" -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD_POLICY" --ignore-not-found --wait=true >/dev/null 2>&1 || true -if ! kubectl -n "$NAMESPACE" run "$SIDE_POD_POLICY" --image=curlimages/curl:8.10.1 --restart=Never \ - --command -- /bin/sh -c ' - # POST a minimal valid PolicySnapshot so any non-2xx response must be - # an auth rejection, not a body-parse rejection. -d sets the body and - # implies POST. - curl -sS -m 5 -o /dev/null -w "%{http_code}" \ - -H "Content-Type: application/json" \ - -d "{\"version\":3,\"policies\":[]}" \ - http://inference-cache-server:8081/policy || echo "curl_failed:$?" - ' >/tmp/policy-probe-create.log 2>&1; then - cat /tmp/policy-probe-create.log >&2 || true - fail "kubectl run $SIDE_POD_POLICY failed; cannot run /policy auth assertion" -fi - -# 90s budget + describe-pod fallback matches the /snapshot probe above — -# the External-backend phase that runs earlier can leave the kubelet busy -# reaping its own Terminating pods, occasionally pushing the new pod's -# container creation above a 30s budget; without the diagnostics dump, -# a timeout would surface as an empty-log failure with no breadcrumb. -for _ in $(seq 1 90); do - phase="$(kubectl -n "$NAMESPACE" get pod "$SIDE_POD_POLICY" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ]; then - break - fi - sleep 1 -done -policy_probe_out="$(kubectl -n "$NAMESPACE" logs "$SIDE_POD_POLICY" 2>/dev/null || true)" -# If the pod never finished, capture its describe output so the failure -# message tells operators why (ImagePullBackOff vs ContainerCreating vs ...). -if [ -z "$policy_probe_out" ]; then - kubectl -n "$NAMESPACE" describe pod "$SIDE_POD_POLICY" >&2 || true -fi -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD_POLICY" --grace-period=0 --force >/dev/null 2>&1 || true - -# Required outcome curl_failed:28 (NetworkPolicy drop) — see /snapshot probe -# above for the full rationale, incl. why 7 (ECONNREFUSED) is NOT accepted (an -# enforcing CNI drops, it does not RST; accepting 7 would let "listener crashed" -# pass) and why a bare 401 now FAILS (it means the NetworkPolicy did not drop -# the connection). 204 (write succeeded unauthenticated) is the regression this -# whole ticket exists to prevent. -case "$policy_probe_out" in - *"curl_failed:28"*) - log "unauthenticated /policy probe dropped by the NetworkPolicy (curl timed out; probe output: $policy_probe_out)" - ;; - "401") - fail "unauthenticated /policy probe reached the listener and got HTTP 401 — the L7 auth middleware answered, but the server NetworkPolicy did NOT drop the connection at L3/L4 (expected curl_failed:28). The NetworkPolicy is missing/broken or the CNI is not enforcing it. Got: $policy_probe_out" - ;; - *) - fail "unauthenticated /policy probe was not dropped by the NetworkPolicy (expected curl_failed:28); got: $policy_probe_out" - ;; -esac - -# --- /probe NetworkPolicy-drop assertion ---------------------------------- -# /probe is the controller-driven functional self-test endpoint; same -# controller ServiceAccount identity as /snapshot and /policy. The complementary -# half — that an UNAUTHENTICATED caller is DROPPED BY THE NETWORKPOLICY — is what -# this section checks: a regression that both wired /probe outside the shared -# auth profile AND broke the NetworkPolicy would let any pod that can reach :8081 -# drive a synthetic round-trip AND, since the CacheBackend reconciler now -# consumes the result to publish FunctionalProbeOK and downgrade Ready, observe -# (or trigger forged) Ready transitions on every managed backend. The side pod -# carries none of the controller labels, so under the enforcing CNI its -# connection to :8081 is dropped and curl exits 28. The body is still a valid -# ProbeRequest so a broken NetworkPolicy surfaces as a clean 401, not a 400. -# Required outcome curl_failed:28; a bare 401 now FAILS. Mirror of the -# /policy probe above. -log "asserting unauthenticated /probe POST from a side pod is dropped by the NetworkPolicy" -SIDE_POD_PROBE="ic-probe-probe" -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD_PROBE" --ignore-not-found --wait=true >/dev/null 2>&1 || true -if ! kubectl -n "$NAMESPACE" run "$SIDE_POD_PROBE" --image=curlimages/curl:8.10.1 --restart=Never \ - --command -- /bin/sh -c ' - # POST a minimal valid ProbeRequest so any non-2xx response must be an - # auth rejection, not a body-parse rejection. - curl -sS -m 5 -o /dev/null -w "%{http_code}" \ - -H "Content-Type: application/json" \ - -d "{\"backend\":\"smoke\",\"model\":\"smoke-model\",\"hashScheme\":\"vllm\"}" \ - http://inference-cache-server:8081/probe || echo "curl_failed:$?" - ' >/tmp/probe-probe-create.log 2>&1; then - cat /tmp/probe-probe-create.log >&2 || true - fail "kubectl run $SIDE_POD_PROBE failed; cannot run /probe auth assertion" -fi - -# 90s budget + describe-pod fallback matches the /snapshot and /policy probes -# above — the surrounding phases (External-backend, audience-binding) can -# leave the kubelet busy reaping Terminating pods. -for _ in $(seq 1 90); do - phase="$(kubectl -n "$NAMESPACE" get pod "$SIDE_POD_PROBE" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ]; then - break - fi - sleep 1 -done -probe_probe_out="$(kubectl -n "$NAMESPACE" logs "$SIDE_POD_PROBE" 2>/dev/null || true)" -if [ -z "$probe_probe_out" ]; then - kubectl -n "$NAMESPACE" describe pod "$SIDE_POD_PROBE" >&2 || true -fi -kubectl -n "$NAMESPACE" delete pod "$SIDE_POD_PROBE" --grace-period=0 --force >/dev/null 2>&1 || true - -# Required outcome curl_failed:28 (NetworkPolicy drop) — see /snapshot probe -# above for the full rationale, incl. why 7 (ECONNREFUSED) is NOT accepted (an -# enforcing CNI drops, it does not RST; accepting 7 would let "listener crashed" -# pass) and why a bare 401 now FAILS. 200 (synthesis ran unauthenticated) is the -# regression this whole section exists to prevent. -case "$probe_probe_out" in - *"curl_failed:28"*) - log "unauthenticated /probe POST dropped by the NetworkPolicy (curl timed out; probe output: $probe_probe_out)" - ;; - "401") - fail "unauthenticated /probe POST reached the listener and got HTTP 401 — the L7 auth middleware answered, but the server NetworkPolicy did NOT drop the connection at L3/L4 (expected curl_failed:28). The NetworkPolicy is missing/broken or the CNI is not enforcing it. Got: $probe_probe_out" - ;; - *) - fail "unauthenticated /probe POST was not dropped by the NetworkPolicy (expected curl_failed:28); got: $probe_probe_out" - ;; -esac - -# --- Audience-binding assertion (/snapshot, /policy, AND /probe) ---------- -# Audience-binding follow-up to the bearer-token gate. The controller pod -# in production mounts THREE ServiceAccount tokens: -# 1. The default automount at /var/run/secrets/kubernetes.io/serviceaccount/token -# — audience = the apiserver. Used by the controller-runtime client. -# 2. A projected volume at /var/run/secrets/inferencecache.io/controller-token/token -# — audience = "inferencecache.io/controller". Used by the CacheIndex -# poller and functional-probe driver (/snapshot + /probe). -# 3. A projected volume at /var/run/secrets/inferencecache.io/policy-token/token -# — audience = "inferencecache.io/policy". Used by the CachePolicy pusher -# (/policy). -# The server passes TokenReviewSpec.Audiences=["inferencecache.io/controller"] -# on /snapshot + /probe reviews, and ["inferencecache.io/policy"] on /policy -# reviews, so a default-audience token MUST come back 401 on all three even -# though the SA identity (controller-manager) would otherwise be admitted. -# -# Why a single probe pod with seven scrapes: it covers the positive path for -# each endpoint's intended audience, the default-audience negative path for all -# three endpoints, and the cross-endpoint "controller token cannot push policy" -# case. Seven small checks, one pod, one assertion: "all outcomes match the -# audience contract." -# -# Scoping — what each smoke gate actually catches (the assertions are -# complementary, not redundant): -# - The CacheIndex assertion earlier (cacheindex/cluster-default.status -# .observedServer populates within ~60s) is what proves the REAL -# controller's controller-token projected-volume manifest, the controller -# binary's BearerTokenPath, the server's flag, and the middleware all agree -# end-to-end. If config/manager/manager.yaml drifts (audience renamed, -# mount path moved, expirationSeconds zeroed), the real poller's -# scrape returns 401 and the CR's observedServer stays empty, failing -# that earlier gate. The CachePolicy adoption assertion earlier is the -# equivalent real-controller check for the policy-token projection. -# - THIS probe asserts only server-side behavior: that each endpoint's -# intended audience-bound token admits and a default-audience token of the -# same SA is rejected on all three endpoints. It uses inline duplicate -# volume specs so it can run even if the controller's manifest is broken -# (which would otherwise mask the server-side check). It does NOT catch -# drift in -# config/manager/manager.yaml; that's the earlier gate's job. -log "asserting audience binding on /snapshot, /policy, and /probe" -PROBE_POD="ic-audience-probe" -kubectl -n "$NAMESPACE" delete pod "$PROBE_POD" --ignore-not-found --wait=true >/dev/null 2>&1 || true - -probe_yaml=$(cat </dev/null || echo "") - policy_token=\$(cat /var/run/secrets/inferencecache.io/policy-token/token 2>/dev/null || echo "") - default_token=\$(cat /var/run/secrets/kubernetes.io/serviceaccount/token 2>/dev/null || echo "") - if [ -z "\$controller_token" ]; then echo "controller_token_missing"; exit 0; fi - if [ -z "\$policy_token" ]; then echo "policy_token_missing"; exit 0; fi - if [ -z "\$default_token" ]; then echo "default_token_missing"; exit 0; fi - # GET /snapshot — controller-audience must 200, default-audience must 401. - sa_ctrl=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$controller_token" "http://inference-cache-server:8081/snapshot" || echo "curl_failed:\$?") - sa_def=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$default_token" "http://inference-cache-server:8081/snapshot" || echo "curl_failed:\$?") - # POST /policy — policy-audience must 204; controller/default audiences must 401. - # Body is a minimal valid PolicySnapshot so any non-2xx is auth-side, not body-parse. - pa_policy=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$policy_token" -H "Content-Type: application/json" -d '{"version":5,"policies":[],"tenants":[]}' "http://inference-cache-server:8081/policy" || echo "curl_failed:\$?") - pa_ctrl=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$controller_token" -H "Content-Type: application/json" -d '{"version":5,"policies":[],"tenants":[]}' "http://inference-cache-server:8081/policy" || echo "curl_failed:\$?") - pa_def=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$default_token" -H "Content-Type: application/json" -d '{"version":5,"policies":[],"tenants":[]}' "http://inference-cache-server:8081/policy" || echo "curl_failed:\$?") - # POST /probe — controller-audience must 200, default-audience must 401. - # Body is a minimal valid ProbeRequest so any non-2xx is auth-side, not body-parse. - pr_ctrl=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$controller_token" -H "Content-Type: application/json" -d '{"backend":"smoke","model":"smoke-model","hashScheme":"vllm"}' "http://inference-cache-server:8081/probe" || echo "curl_failed:\$?") - pr_def=\$(curl -sS -m 5 -o /dev/null -w "%{http_code}" -H "Authorization: Bearer \$default_token" -H "Content-Type: application/json" -d '{"backend":"smoke","model":"smoke-model","hashScheme":"vllm"}' "http://inference-cache-server:8081/probe" || echo "curl_failed:\$?") - echo "snapshot_ctrl=\$sa_ctrl snapshot_def=\$sa_def policy_policy=\$pa_policy policy_ctrl=\$pa_ctrl policy_def=\$pa_def probe_ctrl=\$pr_ctrl probe_def=\$pr_def" - volumeMounts: - - name: controller-token - mountPath: /var/run/secrets/inferencecache.io/controller-token - readOnly: true - - name: policy-token - mountPath: /var/run/secrets/inferencecache.io/policy-token - readOnly: true - volumes: - - name: controller-token - projected: - sources: - - serviceAccountToken: - path: token - audience: inferencecache.io/controller - expirationSeconds: 3600 - - name: policy-token - projected: - sources: - - serviceAccountToken: - path: token - audience: inferencecache.io/policy - expirationSeconds: 3600 -EOF -) -if ! echo "$probe_yaml" | kubectl apply -f - >/tmp/audience-probe-create.log 2>&1; then - cat /tmp/audience-probe-create.log >&2 || true - fail "kubectl apply for $PROBE_POD failed; cannot run audience-binding assertion" -fi - -# Wait for the probe pod to finish; 90s budget + describe-pod fallback -# matches the unauth probes above for the same kubelet-busy-reaping reason. -for _ in $(seq 1 90); do - phase="$(kubectl -n "$NAMESPACE" get pod "$PROBE_POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ]; then - break - fi - sleep 1 -done -audience_probe="$(kubectl -n "$NAMESPACE" logs "$PROBE_POD" 2>/dev/null || true)" -if [ -z "$audience_probe" ]; then - kubectl -n "$NAMESPACE" describe pod "$PROBE_POD" >&2 || true -fi -kubectl -n "$NAMESPACE" delete pod "$PROBE_POD" --grace-period=0 --force >/dev/null 2>&1 || true - -log "audience probe output: $audience_probe" -# Expected outcome line, in order: -# snapshot_ctrl=200 snapshot_def=401 policy_policy=204 policy_ctrl=401 policy_def=401 probe_ctrl=200 probe_def=401 -# Anything else is a regression. curl_failed:28 splits out so an operator -# triaging a red smoke knows whether to look at NetworkPolicy (timeout) vs -# Service/listener (other curl exit). -case "$audience_probe" in - "snapshot_ctrl=200 snapshot_def=401 policy_policy=204 policy_ctrl=401 policy_def=401 probe_ctrl=200 probe_def=401") - log "audience binding verified — controller-audience token admitted on /snapshot and /probe, policy-audience token admitted on /policy, default-audience token rejected everywhere, and controller-audience token rejected on /policy" - ;; - *controller_token_missing*) - fail "probe pod is missing /var/run/secrets/inferencecache.io/controller-token/token — the projected volume did not mount; check the probe-pod manifest above (and config/manager/manager.yaml for the production analog)" - ;; - *policy_token_missing*) - fail "probe pod is missing /var/run/secrets/inferencecache.io/policy-token/token — the projected volume did not mount; check the probe-pod manifest above (and config/manager/manager.yaml for the production analog)" - ;; - *default_token_missing*) - fail "probe pod is missing the default automount; cannot run the audience-binding negative case" - ;; - *"curl_failed:28"*) - fail "audience-binding probe timed out reaching :8081 (curl -m 5 fired). Likely NetworkPolicy regression — does the probe pod still match the controller's component=controller selector? Probe output: $audience_probe" - ;; - *"curl_failed:"*) - fail "audience-binding probe could not connect to the controller-facing listener (curl exited non-zero). Check Service name 'inference-cache-server', port 8081, and that the listener is up. Probe output: $audience_probe" - ;; - *) - fail "audience-binding probe got unexpected outcome: $audience_probe (want 'snapshot_ctrl=200 snapshot_def=401 policy_policy=204 policy_ctrl=401 policy_def=401 probe_ctrl=200 probe_def=401')" - ;; -esac - -# --- /probe functional-self-test result assertion ------------------------- -# The audience-binding section above asserts /probe returned HTTP 200 with the -# controller-audience token, but a deployed handler can also return 200 with a -# per-stage `failed`. This section drives one authenticated /probe call, -# captures the JSON body, and asserts ingest=ok, routing=ok, t2=skipped. -# No T2Prober is wired into the server today, so Stage C always reports -# skipped on a clean install — when one is plumbed in, this assertion -# tightens to t2=ok. A regression that flips ingest or routing to failed -# on a clean install would be a clear signal that the cache-plane -# internal round-trip itself is broken — exactly the class of bug the -# probe exists to catch. -log "asserting authenticated /probe returns ingest=ok, routing=ok, t2=skipped" -PROBE_RESULT_POD="ic-probe-result" -kubectl -n "$NAMESPACE" delete pod "$PROBE_RESULT_POD" --ignore-not-found --wait=true >/dev/null 2>&1 || true -probe_result_yaml=$(cat </dev/null || echo "") - if [ -z "\$controller_token" ]; then echo "controller_token_missing"; exit 0; fi - # Capture body to stdout — the smoke parses ingest/routing/t2 from it. - curl -sS -m 5 -H "Authorization: Bearer \$controller_token" \\ - -H "Content-Type: application/json" \\ - -d '{"backend":"smoke","model":"smoke-model","hashScheme":"vllm"}' \\ - http://inference-cache-server:8081/probe || echo "curl_failed:\$?" - volumeMounts: - - name: controller-token - mountPath: /var/run/secrets/inferencecache.io/controller-token - readOnly: true - volumes: - - name: controller-token - projected: - sources: - - serviceAccountToken: - path: token - audience: inferencecache.io/controller - expirationSeconds: 3600 -EOF -) -if ! echo "$probe_result_yaml" | kubectl apply -f - >/tmp/probe-result-create.log 2>&1; then - cat /tmp/probe-result-create.log >&2 || true - fail "kubectl apply for $PROBE_RESULT_POD failed; cannot run /probe result assertion" -fi -for _ in $(seq 1 90); do - phase="$(kubectl -n "$NAMESPACE" get pod "$PROBE_RESULT_POD" -o jsonpath='{.status.phase}' 2>/dev/null || true)" - if [ "$phase" = "Succeeded" ] || [ "$phase" = "Failed" ]; then - break - fi - sleep 1 -done -probe_result_body="$(kubectl -n "$NAMESPACE" logs "$PROBE_RESULT_POD" 2>/dev/null || true)" -if [ -z "$probe_result_body" ]; then - kubectl -n "$NAMESPACE" describe pod "$PROBE_RESULT_POD" >&2 || true -fi -kubectl -n "$NAMESPACE" delete pod "$PROBE_RESULT_POD" --grace-period=0 --force >/dev/null 2>&1 || true -log "probe result body: $probe_result_body" -case "$probe_result_body" in - *controller_token_missing*) - fail "probe-result pod is missing /var/run/secrets/inferencecache.io/controller-token/token" - ;; - *"curl_failed:"*) - fail "probe-result curl failed: $probe_result_body" - ;; -esac -# Parse the three stage values; reject anything that isn't the expected -# default posture (ingest=ok, routing=ok, t2=skipped). The default-install -# CacheBackend has no engine pods reporting state, but the probe synthesizes -# its own — so Stage A + B must always pass on a clean install regardless of -# workload. Stage C is "skipped" because no T2Prober is wired in this revision. -case "$probe_result_body" in - *'"ingest":"ok"'*'"routing":"ok"'*'"t2":"skipped"'*) - log "probe result matches expected default posture (ingest=ok, routing=ok, t2=skipped)" - ;; - *) - fail "probe result does not match expected default posture; want ingest=ok routing=ok t2=skipped, got: $probe_result_body" - ;; -esac - -# --- opt-in gRPC TLS overlay verification ---------------------------------- -# config/default is plaintext; Service TLS is an opt-in overlay -# (config/overlays/server-tls = config/default + the config/server/tls -# component). Apply it on top of the running install, which patches the server -# Deployment with --tls-cert-file/--tls-key-file + the cert-manager Secret -# volume and ships the Issuer + Certificate. After the rollout, verify the -# cert-manager-issued chain + Service-FQDN SAN actually authenticate the server -# (not just encrypt): pull ca.crt from the serving Secret and run -# `grpcurl -cacert -authority ` (grpcurl uses -authority -# as the verification name even though the port-forward terminates at -# localhost). A wrong authority must fail, and plaintext must be rejected. -log "verifying opt-in TLS overlay (config/overlays/server-tls)" -kubectl apply -k "$tmpdir/config/overlays/server-tls" >/dev/null \ - || fail "kubectl apply -k config/overlays/server-tls failed" -# The patched pod stays Pending until cert-manager mints the Secret from the -# Certificate the overlay just applied; rollout status blocks until it's served. -if ! kubectl -n "$NAMESPACE" rollout status deploy/inference-cache-server --timeout=150s; then - kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/component=server -o wide || true - kubectl -n "$NAMESPACE" describe certificate inference-cache-server-serving-cert || true - fail "server Deployment did not roll out with TLS within 150s (cert-manager Secret not minted?)" -fi - -# Re-establish the port-forward against the freshly rolled (TLS) pod; the old -# forward pointed at the now-terminated plaintext pod. -kill "$pf_pid" 2>/dev/null || true -TLS_LOCAL_PORT="$((GRPC_LOCAL_PORT + 1))" -kubectl -n "$NAMESPACE" port-forward svc/inference-cache-server "$TLS_LOCAL_PORT:9090" \ - >"$LOG_DIR/port-forward-tls.log" 2>&1 & -pf_pid=$! -tls_ready=0 -for _ in $(seq 1 30); do - if grpcurl -insecure -max-time 2 "localhost:$TLS_LOCAL_PORT" list >/dev/null 2>&1; then - tls_ready=1 - break - fi - sleep 1 -done -# Assert the TLS port-forward actually came up before the real checks run — an -# explicit failure here points at the forward / TLS listener rather than letting -# the -cacert assertion below fail with a murkier "connection refused". -if [ "$tls_ready" != "1" ]; then - cat "$LOG_DIR/port-forward-tls.log" >&2 || true - fail "TLS port-forward to :9090 never accepted a connection (grpcurl -insecure failed for 30s after the overlay rollout)" -fi - -if grpcurl -plaintext -max-time 5 "localhost:$TLS_LOCAL_PORT" list \ - >"$LOG_DIR/grpcurl-tls-plaintext.out" 2>&1; then - cat "$LOG_DIR/grpcurl-tls-plaintext.out" >&2 || true - fail "expected plaintext to be REJECTED once the TLS overlay is applied" -fi -ca_file="$LOG_DIR/server-ca.crt" -kubectl -n "$NAMESPACE" get secret inference-cache-server-tls \ - -o jsonpath='{.data.ca\.crt}' 2>/dev/null | base64 -d > "$ca_file" || true -if [ ! -s "$ca_file" ]; then - kubectl -n "$NAMESPACE" get secret inference-cache-server-tls -o yaml || true - fail "serving Secret inference-cache-server-tls has no usable ca.crt — clients could not authenticate the server (encryption only)" -fi -server_fqdn="inference-cache-server.${NAMESPACE}.svc.cluster.local" -if ! grpcurl -cacert "$ca_file" -authority "$server_fqdn" -max-time 5 \ - "localhost:$TLS_LOCAL_PORT" list >"$LOG_DIR/grpcurl-cacert-list.out" 2>&1; then - cat "$LOG_DIR/grpcurl-cacert-list.out" >&2 || true - fail "expected TLS verification with the cert-manager CA against $server_fqdn to succeed" -fi -if grpcurl -cacert "$ca_file" -authority "wrong.example.invalid" -max-time 5 \ - "localhost:$TLS_LOCAL_PORT" list >"$LOG_DIR/grpcurl-cacert-badname.out" 2>&1; then - cat "$LOG_DIR/grpcurl-cacert-badname.out" >&2 || true - fail "expected TLS verification with a wrong authority to FAIL (the Service-FQDN SAN must be enforced)" -fi -log "opt-in TLS overlay OK: plaintext rejected; cert-manager CA verifies the server cert for $server_fqdn (wrong name rejected)" - -# Backward-compatibility: the EXISTING call pattern must work UNCHANGED over -# TLS. Re-run the same LookupRoute(unknown model) the plaintext phase ran (phase -# 7) and assert the identical fail-open result (reason_code=NO_HINT) — proving -# TLS is a pure transport wrapper that does not alter the gRPC contract or -# handler behavior, so a client only swaps plaintext creds for TLS creds. -# Reflection first, proto-file fallback (same priority order as the plaintext -# probe in grpcurl_lookup_route). -tls_lookup_payload='{"modelId":"install-smoke-unknown"}' -tls_lookup_resp="$(grpcurl -cacert "$ca_file" -authority "$server_fqdn" -max-time 5 -d "$tls_lookup_payload" \ - "localhost:$TLS_LOCAL_PORT" inferencecache.v1alpha1.InferenceCache/LookupRoute 2>"$LOG_DIR/grpcurl-tls-lookup.err" \ - || grpcurl -cacert "$ca_file" -authority "$server_fqdn" -max-time 5 \ - -import-path proto -proto inferencecache/v1alpha1/inferencecache.proto -d "$tls_lookup_payload" \ - "localhost:$TLS_LOCAL_PORT" inferencecache.v1alpha1.InferenceCache/LookupRoute 2>>"$LOG_DIR/grpcurl-tls-lookup.err")" -if [ -z "$tls_lookup_resp" ]; then - cat "$LOG_DIR/grpcurl-tls-lookup.err" >&2 || true - fail "LookupRoute over TLS returned no response — the existing call pattern broke once TLS was enabled" -fi -if ! has_reason_code "$tls_lookup_resp" "NO_HINT"; then - fail "LookupRoute over TLS did not return the fail-open NO_HINT the plaintext path returns (TLS altered handler behavior): $tls_lookup_resp" -fi -log "existing call pattern intact over TLS: LookupRoute(unknown model) → NO_HINT, identical to the plaintext phase" - -# --- kernel-check init-container injection shape (assertion 14) ------------ -# Block 1: prove the mutating pod webhook injects lmcache-kernel-check when -# auto mode fires (GPU-requesting container). A dedicated LMCache CacheBackend -# (kc-inject) is created in the kernel-check smoke namespace so the webhook -# can resolve a matching backend at pod admission time. The engine pod carries -# the backend's engineSelector label (app=kc-inject-engine), requests -# nvidia.com/gpu → auto mode injects the init container. The kind node has no -# GPU, so the pod stays Pending — that is expected and correct; the webhook -# runs at admission, before scheduling. Only the SPEC is inspected here (no -# status, no pod running). -log "asserting lmcache-kernel-check init-container injection shape (assertion 14)" -# Start from a clean namespace: on a rerun against a kept cluster, leftover pods -# would be UPDATED by `kubectl apply` (the mutating webhook only fires on -# CREATE), so the assertions could pass/fail on stale injected specs. Delete and -# recreate so every fixture pod is created fresh and re-admitted. -kubectl delete namespace "$KERNEL_CHECK_SMOKE_NS" --ignore-not-found=true --wait=true >/dev/null 2>&1 || true -kubectl create namespace "$KERNEL_CHECK_SMOKE_NS" --dry-run=client -o yaml \ - | kubectl apply -f - >/dev/null - -# Apply the CacheBackend first so status.endpoint is published before the -# engine pod is admitted (the webhook fail-opens when endpoint is absent). -KC_INJECT_CB="kc-inject" -KC_INJECT_POD="kc-inject-probe" -kubectl apply -f - >/dev/null </dev/null || true)" - if [ -n "$kc_inject_endpoint" ]; then break; fi - sleep 2 -done -if [ -z "$kc_inject_endpoint" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_INJECT_CB" -o yaml || true - fail "$KC_INJECT_CB status.endpoint not published within 60s; cannot exercise the webhook injection (endpoint must be set before engine pod CREATE)" -fi - -kubectl apply -f - >/dev/null </dev/null || true)" - if [ "$kc_injected_name" = "lmcache-kernel-check" ]; then break; fi - sleep 2 -done -if [ "$kc_injected_name" != "lmcache-kernel-check" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get pod "$KC_INJECT_POD" -o yaml || true - fail "lmcache-kernel-check init container not injected into GPU-requesting engine pod after 30s (webhook auto mode did not fire)" -fi - -# Assert the init container's image equals the engine container's image -# (busybox:1.36). The adapter must copy the engine image so no extra pull occurs. -kc_init_image="$(kubectl -n "$KERNEL_CHECK_SMOKE_NS" get pod "$KC_INJECT_POD" \ - -o jsonpath='{.spec.initContainers[?(@.name=="lmcache-kernel-check")].image}' 2>/dev/null || true)" -if [ "$kc_init_image" != "busybox:1.36" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get pod "$KC_INJECT_POD" -o yaml || true - fail "lmcache-kernel-check init container image=$kc_init_image, want busybox:1.36 (adapter must reuse the engine container image)" -fi -log "lmcache-kernel-check init container injected; image=$kc_init_image (matches engine container — no extra pull)" - -# --- kernel-check report-only FAIL condition path (assertion 15) ----------- -# Block 2: prove the report-only FAIL path is fail-open and surfaces -# EngineKernelsHealthy=False/KernelLoadFailed. A dedicated managed LMCache -# CacheBackend (kc-cond) is annotated report-only. The matching engine pod -# uses python:3.11-slim: the init container runs the kernel-check script, which -# calls find_spec("lmcache") → None → emits "FAIL: lmcache not importable" to -# /dev/termination-log and exits 0 (fail-open; STRICT unset). The main container -# starts normally, so the pod reaches Ready — proving fail-open semantics. -# The C2 reconciler reads the termination message from -# status.initContainerStatuses and publishes EngineKernelsHealthy=False / -# reason=KernelLoadFailed on the CacheBackend. -# -# python:3.11-slim was chosen deliberately: it has python3 (so the init container -# runs successfully, producing our FAIL: message) but does NOT have lmcache -# installed (find_spec returns None → the FAIL: branch fires). Using a non-python -# image (e.g. pause) would produce a 127 exit / KernelCheckError, not -# KernelLoadFailed, which would break the condition assertion. -log "asserting report-only FAIL path: EngineKernelsHealthy=False/KernelLoadFailed (assertion 15)" -KC_COND_CB="kc-cond" -KC_COND_ENGINE_POD="kc-cond-engine" -kubectl apply -f - >/dev/null </dev/null || true)" - if [ -n "$kc_cond_endpoint" ]; then break; fi - sleep 2 -done -if [ -z "$kc_cond_endpoint" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_COND_CB" -o yaml || true - fail "$KC_COND_CB status.endpoint not published within 60s; cannot exercise the report-only condition path" -fi - -# Apply the engine pod. python:3.11-slim is small (~50 MB) and always present on -# Docker Hub, so this phase does not pay a multi-GB pull. No GPU request: the -# report-only mode injects regardless of GPU (the annotation overrides auto mode). -kubectl apply -f - >/dev/null </dev/null 2>&1; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get pod "$KC_COND_ENGINE_POD" -o yaml || true - fail "$KC_COND_ENGINE_POD did not become Ready within ${KERNEL_CHECK_POD_TIMEOUT}s — report-only did not fail-open (init container may have exited non-zero, or the image pull stalled)" -fi -log "$KC_COND_ENGINE_POD is Ready — report-only mode did not block the engine pod" - -# Poll for EngineKernelsHealthy=False on the CacheBackend. The reconciler reads -# the init container's termination message from status.initContainerStatuses on -# the matched pod; it fires on the next reconcile after the init container -# completes. The budget absorbs one RequeueAfter cycle + pod-list round-trip. -log "waiting up to ${KERNEL_CHECK_COND_TIMEOUT}s for EngineKernelsHealthy=False on $KC_COND_CB" -deadline=$(($(date +%s) + KERNEL_CHECK_COND_TIMEOUT)) -kc_cond_status="" -while [ "$(date +%s)" -lt "$deadline" ]; do - kc_cond_status="$(kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_COND_CB" \ - -o jsonpath='{.status.conditions[?(@.type=="EngineKernelsHealthy")].status}' 2>/dev/null || true)" - if [ "$kc_cond_status" = "False" ]; then break; fi - sleep 3 -done -if [ "$kc_cond_status" != "False" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_COND_CB" -o yaml || true - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get pod "$KC_COND_ENGINE_POD" -o yaml || true - fail "EngineKernelsHealthy condition status=$kc_cond_status on $KC_COND_CB after ${KERNEL_CHECK_COND_TIMEOUT}s; want False (reconciler did not read the FAIL: termination message)" -fi - -kc_cond_reason="$(kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_COND_CB" \ - -o jsonpath='{.status.conditions[?(@.type=="EngineKernelsHealthy")].reason}' 2>/dev/null || true)" -if [ "$kc_cond_reason" != "KernelLoadFailed" ]; then - kubectl -n "$KERNEL_CHECK_SMOKE_NS" get cb "$KC_COND_CB" -o yaml || true - fail "EngineKernelsHealthy reason=$kc_cond_reason on $KC_COND_CB; want KernelLoadFailed (FAIL: termination message must map to this reason)" -fi -log "EngineKernelsHealthy=False/KernelLoadFailed on $KC_COND_CB — report-only FAIL path wired end-to-end" - -# The validating webhook must reject an invalid lmcache-kernel-check annotation -# value: a typo (e.g. "strcit") would otherwise silently fall back to report-only -# and disable the fail-closed strict gate. Proves the rule on the real install. -log "asserting an invalid lmcache-kernel-check annotation is rejected at admission" -kc_badannot_yaml="$(cat <&1)"; then - echo "$kc_badannot_out" - fail "CacheBackend with an invalid lmcache-kernel-check annotation was admitted; the validating webhook should reject it" -fi -if ! grep -q "must be one of" <<<"$kc_badannot_out"; then - echo "$kc_badannot_out" - fail "invalid kernel-check annotation rejected, but not by the expected rule (missing 'must be one of' message)" -fi -log "invalid lmcache-kernel-check annotation rejected at admission" - -# Cleanup: drop the kernel-check smoke namespace. -kubectl delete namespace "$KERNEL_CHECK_SMOKE_NS" \ - --wait=false --ignore-not-found=true >/dev/null 2>&1 || true - -# --- sample-manifest apply-clean backstop ---------------------------------- -# Every YAML under config/samples/ must apply cleanly against the running -# install. Operators copy these as their first-contact recipe; a sample that -# rejects at admission (names a (runtime, type) pair no shipped adapter -# supports, populates a field the schema no longer accepts) burns trust on -# first contact. -# -# Envtest-level sample validation already lives in `make verify-samples` -# (it runs every sample through admission against an in-process apiserver + -# the CacheBackend webhook). This phase is the live-kind-cluster complement: -# it exercises the SAME samples against the real default install via -# server-side dry-run, so it additionally catches admission-path failures -# that envtest's self-managed apiserver + webhook certs mask — the CacheBackend -# validating webhook being unreachable or mis-wired on a real cluster -# (cert-manager-injected caBundle, Service routing, failurePolicy), and the -# CRDs as actually installed by `kubectl apply -k config/default` rather than -# from envtest's CRDDirectoryPaths. Belt-and-suspenders against sample drift -# in the actually-deployed scenario. -# -# Scope note: --dry-run=server stops at apiserver admission. It does NOT -# create CRs, drive controllers, write status, or hit the /policy + /snapshot -# HTTP endpoints — so it exercises no NetworkPolicy or status-write RBAC (the -# earlier per-CRD behavioral phases cover those). This is a CRD + admission- -# wiring backstop, nothing more. -# -# Runs LAST: the per-CRD phases above (External backend, CachePolicy push, -# binding signals) assert behavioral regressions first; this generic -# apply-clean loop is the catch-all backstop. Server-side dry-run exercises -# CRD structural validation AND the validating admission webhook without -# persisting any CRs (nothing to clean up afterwards); the only cluster side -# effect is one transient namespace, created below and deleted at the end so -# namespaced samples have a target. Spins up no engine pods (admission-level -# only, no traffic). -# -# Honors the same opt-out as `make verify-samples`: a sample whose top-of-file -# comment block contains a line equal to `# verify-samples: skip` is reported -# as SKIP and not applied. Keeping the two gates' sample sets in lockstep -# means a sample intentionally excluded from one is excluded from both. -log "asserting every config/samples/ manifest applies cleanly against the live install (server dry-run)" -SAMPLE_APPLY_NS="${SAMPLE_APPLY_NS:-ic-sample-apply}" -kubectl create namespace "$SAMPLE_APPLY_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -# sample_skip_marker / has_skip_marker mirror hack/verify-samples' -# hasSkipMarker: scan only the leading comment block (blank + '#'-prefixed -# lines), stop at the first non-comment line, and match the marker exactly -# after trimming surrounding whitespace. Defined here (not at the top) to keep -# the backstop self-contained. -sample_skip_marker="# verify-samples: skip" -has_skip_marker() { - local f="$1" line trimmed - while IFS= read -r line || [ -n "$line" ]; do - trimmed="${line#"${line%%[![:space:]]*}"}" # ltrim - trimmed="${trimmed%"${trimmed##*[![:space:]]}"}" # rtrim - [ -z "$trimmed" ] && continue - case "$trimmed" in - "#"*) [ "$trimmed" = "$sample_skip_marker" ] && return 0 ;; - *) return 1 ;; # first non-comment line: marker can no longer appear - esac - done < "$f" - return 1 -} - -# Enumerate the same sample set as `make verify-samples` (hack/verify-samples' -# listSamples): every regular *.yaml / *.yml under config/samples, recursively, -# sorted for deterministic output. Tracking its selection keeps the two gates -# in lockstep — a future .yml or subdirectory sample can't be covered by the -# envtest gate yet silently skipped by this live-cluster one. (config/samples -# holds only regular files; symlinked samples — which `find -type f` skips and -# Go's filepath.Walk would include — are not used here, so the sets match.) -# -# Materialize the list FIRST, with an explicit error check, rather than piping -# find straight into the loop via process substitution: a process -# substitution's exit status is discarded, so `set -o pipefail` can't observe a -# find failure, and a traversal that errored after emitting some files would -# slip past the zero-match guard below as partial coverage. Failing here means -# the coverage gate never silently passes on a partial walk. (pipefail is set -# at the top of the script, so the command substitution sees find's status.) -sample_list="$(find config/samples -type f \( -name '*.yaml' -o -name '*.yml' \) | sort)" \ - || fail "could not enumerate config/samples manifests (find failed) — refusing to report partial sample coverage" -sample_ok=0 -sample_skip=0 -sample_fail=0 -while IFS= read -r f; do - [ -n "$f" ] || continue # skip the lone empty line an empty here-string yields - if has_skip_marker "$f"; then - log " SKIP $f (opt-out: $sample_skip_marker)" - sample_skip=$((sample_skip + 1)) +log "checking current samples against the live CRDs and admission webhooks" +sample_namespace="${SMOKE_NAMESPACE}-samples" +kubectl create namespace "$sample_namespace" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +sample_count=0 +while IFS= read -r sample; do + if awk ' + /^[[:space:]]*$/ { next } + /^[[:space:]]*#/ { if ($0 ~ /#[[:space:]]*verify-samples:[[:space:]]*skip[[:space:]]*$/) found=1; next } + { exit } + END { exit(found ? 0 : 1) } + ' "$sample"; then continue fi - # cacheindex is cluster-scoped; -n is a harmless no-op for it. Everything - # else under config/samples is namespace-scoped, so a dedicated namespace - # keeps each sample's default ObjectMeta from colliding with earlier phases' - # fixtures. --dry-run=server persists nothing, so no teardown is needed. - # --request-timeout bounds a hung apply (mirrors verify-samples' - # perSampleTimeout) so a stuck apiserver/admission webhook fails THIS sample - # fast — surfacing its filename via the rejection branch below — instead of - # stalling CI until the workflow timeout with no in-flight breadcrumb. - if kubectl apply --dry-run=server --request-timeout=30s -n "$SAMPLE_APPLY_NS" -f "$f" \ - >/tmp/sample-dry-run.log 2>&1; then - log " OK $f" - sample_ok=$((sample_ok + 1)) - else - echo "[install-smoke] sample $f did not apply cleanly:" >&2 - cat /tmp/sample-dry-run.log >&2 - sample_fail=$((sample_fail + 1)) - fi -done <<< "$sample_list" -# Operator-facing admission signal: the (sglang, LMCache) sample MUST now admit -# CLEANLY under real server-side admission — the adapter renders the working -# LMCache MP-mode data plane (node-local MP-worker sidecar + config-file wire → the -# managed Redis L2), so the old "offload misconfigured (lm:// vs MP)" advisory is -# gone. --dry-run=server reaches admission and persists nothing. The loop above only -# asserts the sample parses/applies; this asserts it admits without the obsolete -# warning through a real apply. Runs in $SAMPLE_APPLY_NS (still fresh — the loop's -# dry-runs persisted nothing) BEFORE its delete below, so this is a clean CREATE. -sglang_sample="config/samples/cachebackend-sglang.yaml" -if [ -f "$sglang_sample" ]; then - # `if cmd; then` keeps `set -e` from aborting on a rejected apply AND distinguishes - # the failure modes: a NON-zero exit means admission rejected the sample; a zero - # exit means it admitted, and we then require the obsolete warning to be ABSENT - # (its presence would mean the working wire didn't remove it). Match the retired - # warning's EXACT text, not loose fragments like "misconfigured" / "MP mode": those - # appear in ordinary prose, so an unrelated future warning would fail this gate for - # the wrong reason. - sglang_retired_warn="SGLang+LMCache offload misconfigured: SGLang needs LMCache MP mode, not this lm:// server" - if sglang_warn_out="$(kubectl apply --dry-run=server --request-timeout=30s -n "$SAMPLE_APPLY_NS" -f "$sglang_sample" 2>&1)"; then - case "$sglang_warn_out" in - *"$sglang_retired_warn"*) - printf '%s\n' "$sglang_warn_out" - fail "(sglang, LMCache) still emits the obsolete offload-misconfigured warning — the MP-mode data plane should have removed it" ;; - *) - log "(sglang, LMCache) sample admits cleanly (working MP-mode data plane wired, no obsolete warning)" ;; - esac - else - printf '%s\n' "$sglang_warn_out" - fail "(sglang, LMCache) sample did not apply cleanly under --dry-run=server (admission rejected it)" - fi -else - fail "$sglang_sample missing — the SGLang admission assertion cannot run; a rename/deletion must not silently drop this operator-facing gate" -fi - -kubectl delete namespace "$SAMPLE_APPLY_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true - -# Guard against the backstop silently becoming a no-op if config/samples ends -# up empty or unreadable. The recursive find above tracks `make verify-samples`, -# so layout changes (subdirs, .yml) stay covered; this only catches the -# degenerate "no samples at all" case. -if [ "$((sample_ok + sample_skip + sample_fail))" -eq 0 ]; then - fail "no *.yaml/*.yml manifests found under config/samples — the apply-clean backstop covered nothing (is the sample directory missing or empty?)" -fi -if [ "$sample_fail" -ne 0 ]; then - fail "$sample_fail config/samples/ manifest(s) did not apply cleanly against the live install — see the rejection output above" -fi -log "all config/samples/ manifests applied cleanly ($sample_ok ok, $sample_skip skipped; server dry-run)" - -# --- inferencecache doctor CLI assertion ----------------------------------- -# The operator-facing `inferencecache doctor` CLI must run end-to-end against a -# real install, not just envtest. Build it and run the config-only checks (no -# live server probe required) against a freshly-applied CacheBackend, asserting -# it emits the documented JSON envelope, actually inspected the backend (a CB0xx -# finding), and that its process exit code matches the reported summary.exitCode -# — the CI-gating contract operators rely on. -log "asserting 'inferencecache doctor' runs against the live install" -mkdir -p "$LOG_DIR" -DOCTOR_BIN="$LOG_DIR/inferencecache" -if ! go build -o "$DOCTOR_BIN" ./cmd/inferencecache >"$LOG_DIR/doctor-build.log" 2>&1; then - cat "$LOG_DIR/doctor-build.log" >&2 || true - fail "could not build cmd/inferencecache for the doctor smoke assertion" -fi -DOCTOR_NS=doctor-smoke -kubectl create namespace "$DOCTOR_NS" >/dev/null 2>&1 || true -kubectl apply -n "$DOCTOR_NS" -f config/samples/cache_v1alpha1_cachebackend.yaml >/dev/null + kubectl -n "$sample_namespace" apply --dry-run=server -f "$sample" >/dev/null \ + || fail "sample failed live server-side admission: $sample" + sample_count=$((sample_count + 1)) +done < <(find config/samples -type f \( -name '*.yaml' -o -name '*.yml' \) | sort) +[ "$sample_count" -gt 0 ] || fail "no samples were checked" + +log "checking non-LMCache control-plane APIs and engine-local backends" +for sample in \ + config/samples/cache_v1alpha1_cachepolicy.yaml \ + config/samples/cache_v1alpha1_cachetenant.yaml \ + config/samples/cache_v1alpha1_prompttemplate.yaml \ + config/samples/cache_v1alpha1_pdtopology.yaml \ + config/samples/cachebackend-events-only.yaml \ + config/samples/cachebackend-sglang-hicache.yaml; do + kubectl -n "$sample_namespace" apply -f "$sample" >/dev/null +done +kubectl -n "$sample_namespace" get cachepolicy cachepolicy-sample >/dev/null +kubectl -n "$sample_namespace" get cachetenant cachetenant-sample >/dev/null +kubectl -n "$sample_namespace" get prompttemplate prompttemplate-sample >/dev/null +kubectl -n "$sample_namespace" get pdtopology pdtopology-sample >/dev/null +for engine_local in cachebackend-events-only sglang-hicache; do + if kubectl -n "$sample_namespace" get deployment "$engine_local" >/dev/null 2>&1 || \ + kubectl -n "$sample_namespace" get service "$engine_local" >/dev/null 2>&1; then + fail "$engine_local unexpectedly provisioned a backend workload" + fi +done + +log "checking the operator-facing doctor CLI against the live install" +doctor_bin="$tmpdir/inferencecache" +go build -o "$doctor_bin" ./cmd/inferencecache doctor_rc=0 -"$DOCTOR_BIN" doctor --config-only --namespace "$DOCTOR_NS" --output json --no-color \ +"$doctor_bin" doctor --config-only --namespace "$SMOKE_NAMESPACE" --output json --no-color \ >"$LOG_DIR/doctor.json" 2>"$LOG_DIR/doctor.err" || doctor_rc=$? -kubectl delete namespace "$DOCTOR_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true -case "$doctor_rc" in - 0|1|2) : ;; - *) cat "$LOG_DIR/doctor.json" "$LOG_DIR/doctor.err" >&2 || true - fail "inferencecache doctor exited $doctor_rc (want a CI-gating 0/1/2)" ;; -esac -if ! grep -q '"summary"' "$LOG_DIR/doctor.json" || ! grep -q '"findings"' "$LOG_DIR/doctor.json"; then - cat "$LOG_DIR/doctor.json" >&2 || true - fail "inferencecache doctor did not emit the expected JSON envelope (summary + findings)" -fi -if ! grep -q '"code": "CB0' "$LOG_DIR/doctor.json"; then - cat "$LOG_DIR/doctor.json" >&2 || true - fail "inferencecache doctor produced no CacheBackend (CB0xx) finding despite an applied backend" -fi -if ! grep -q "\"exitCode\": $doctor_rc" "$LOG_DIR/doctor.json"; then - cat "$LOG_DIR/doctor.json" >&2 || true - fail "doctor process exit ($doctor_rc) does not match the reported summary.exitCode" -fi -log "inferencecache doctor ran against the live install (exit $doctor_rc; JSON envelope + CB finding present)" - -# --- legacy IP managed Mooncake compatibility smoke ------------------------ -# INTENTIONAL LEGACY FIXTURE: Mooncake-through-LMCache/IP remains implemented -# until Phase 7, but is not a current production path or sample. This section -# keeps compatibility coverage without presenting it as a recommended backend. -# CacheBackend{runtime: VLLM, type: LMCache, -# remoteStorage.provider: Mooncake} is still an operator-facing alpha surface, so it needs -# a real-install assertion, not just unit/envtest. The kvcacheai/mooncake image -# is intentionally NOT pulled here (heavy, and its entrypoint/ports are pending -# reference-stack validation); instead a busybox stand-in named `mooncake_master` -# accepts TCP on the RPC port so the controller-rendered readiness probe passes -# and the managed Deployment reaches Available. That proves the REAL installed -# controller reconciles the canonical provider through ResolveCacheServer into -# a healthy workload + the mooncakestore:// RPC endpoint in status — the -# operator-visible contract envtest can't fully exercise (real install bundle + -# real controller image). The real engine-over-mooncakestore:// path stays for -# the reference stack. -MOONCAKE_SMOKE_NS="${MOONCAKE_SMOKE_NS:-mooncake-smoke}" -MOONCAKE_MASTER_IMAGE="${MOONCAKE_MASTER_IMAGE:-install-smoke-mooncake-master:$TAG}" -MOONCAKE_CB_NAME="cachebackend-mooncake" - -log "building lightweight mooncake_master stand-in image=$MOONCAKE_MASTER_IMAGE" -mc_ctx="$(mktemp -d "$tmpdir/mooncake-master-context.XXXXXX")" -cat >"$mc_ctx/mooncake_master" <<'EOF' -#!/bin/sh -# Stand-in: ignore all flags (--rpc_port=..., metadata/metrics ports) and just -# accept TCP on the RPC port so the TCP-socket readiness probe passes. -while true; do - nc -l -p 50051 >/dev/null 2>&1 || sleep 1 -done -EOF -cat >"$mc_ctx/Dockerfile" <<'EOF' -FROM busybox:1.36 -COPY mooncake_master /usr/local/bin/mooncake_master -RUN chmod +x /usr/local/bin/mooncake_master -EOF -docker build -t "$MOONCAKE_MASTER_IMAGE" "$mc_ctx" >/dev/null -"$KIND" load docker-image "$MOONCAKE_MASTER_IMAGE" --name "$KIND_CLUSTER" >/dev/null - -log "creating namespace $MOONCAKE_SMOKE_NS" -kubectl create namespace "$MOONCAKE_SMOKE_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null - -mc_cb_tmp="$(mktemp "$tmpdir/mooncake-cb.XXXXXX")" -cat >"$mc_cb_tmp" <<'EOF' -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: cachebackend-mooncake -spec: - runtime: VLLM - type: LMCache - deploymentKind: Deployment - replicas: 1 - integration: - role: ReadWrite - engineHostNetwork: true - engineSelector: - matchLabels: - app.kubernetes.io/name: vllm - observation: - modelID: meta-llama/Meta-Llama-3-8B-Instruct - remoteStorage: - provider: Mooncake - ownership: Managed - mooncake: - image: docker.io/kvcacheai/mooncake:0.3.11.post1 -EOF -mc_escaped_image="$(printf '%s' "$MOONCAKE_MASTER_IMAGE" | sed 's/[&|\\]/\\&/g')" -if ! grep -q '^ image: docker.io/kvcacheai/mooncake:0.3.11.post1$' "$mc_cb_tmp"; then - fail "legacy inline Mooncake fixture no longer carries remoteStorage.mooncake.image" -fi -sed -i.bak "s|^ image: docker.io/kvcacheai/mooncake:0.3.11.post1$| image: $mc_escaped_image|g" "$mc_cb_tmp" -rm -f "${mc_cb_tmp}.bak" -if ! grep -Fq " image: $MOONCAKE_MASTER_IMAGE" "$mc_cb_tmp"; then - fail "fixture: failed to replace canonical remoteStorage.mooncake.image with the smoke stand-in" -fi - -log "applying Mooncake CacheBackend" -# Mooncake's transfer engine is a peer-to-peer mesh, so ENGINE pods must run with -# hostNetwork too. That rewrites a pod the operator owns, so it is opt-in via -# spec.integration.engineHostNetwork rather than injected. Both halves of that -# contract are asserted against the real install, because both fail silently: -# -# 1. WITHOUT the opt-in, admission must WARN — otherwise the operator receives a -# backend that reports Ready and transfers zero KV, visible only as a flat -# cache-hit graph. -# 2. WITH the opt-in (what the sample ships), the warning must be SILENT — a -# warning that keeps firing after the gap is closed trains operators to -# ignore warnings. -# Guard the fixture, not just the behaviour: if the field is renamed or -# re-indented, the sed below silently no-ops and the first assertion then fails -# with "did not warn" — blaming the webhook for a broken test fixture. -if ! grep -q '^ engineHostNetwork: true$' "$mc_cb_tmp"; then - fail "legacy inline Mooncake fixture no longer carries 'engineHostNetwork: true' at the expected indent; the no-opt-in copy would be a no-op" -fi -mc_nooptin_tmp="$(mktemp "$tmpdir/mooncake-cb-nooptin.XXXXXX")" -sed 's/^ engineHostNetwork: true$//' "$mc_cb_tmp" >"$mc_nooptin_tmp" -mc_warn_out="$(kubectl -n "$MOONCAKE_SMOKE_NS" apply --dry-run=server -f "$mc_nooptin_tmp" 2>&1)" -case "$mc_warn_out" in - *"spec.integration.engineHostNetwork=true"*) - log "Mooncake without the opt-in emits the engine-hostNetwork admission warning" ;; - *) - printf '%s\n' "$mc_warn_out" - fail "Mooncake without spec.integration.engineHostNetwork did not warn (the incomplete data plane must stay loud)" ;; -esac - -mc_apply_out="$(kubectl -n "$MOONCAKE_SMOKE_NS" apply -f "$mc_cb_tmp" 2>&1)" -case "$mc_apply_out" in - *"engineHostNetwork"*) - printf '%s\n' "$mc_apply_out" - fail "Mooncake WITH the opt-in still warned about engineHostNetwork; the warning must go quiet once the gap is closed" ;; - *) - log "Mooncake with the opt-in applies without the engine-hostNetwork warning" ;; -esac - -# Pin the fixture to the retained legacy hierarchy. -mc_runtime="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.runtime}')" -mc_type="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.type}')" -mc_provider="$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o jsonpath='{.spec.remoteStorage.provider}')" -if [ "$mc_runtime" != "VLLM" ] || [ "$mc_type" != "LMCache" ] || [ "$mc_provider" != "Mooncake" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake compatibility fixture changed shape: runtime=$mc_runtime type=$mc_type remoteStorage.provider=$mc_provider" -fi -log "legacy Mooncake CacheBackend admitted (runtime=VLLM, type=LMCache, provider=Mooncake)" - -# Reuses SAMPLE_ENDPOINT_TIMEOUT deliberately: the reconcile-to-status.endpoint -# latency is a per-managed-backend property (the reconciler publishes it from -# the live Service), identical for the LMCache and Mooncake managed paths — no -# Mooncake-specific tunable is warranted. -log "waiting up to ${SAMPLE_ENDPOINT_TIMEOUT}s for Mooncake status.endpoint" -mc_deadline=$(($(date +%s) + SAMPLE_ENDPOINT_TIMEOUT)) -mc_endpoint="" -while [ "$(date +%s)" -lt "$mc_deadline" ]; do - mc_endpoint=$(kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" \ - -o jsonpath='{.status.endpoint}' 2>/dev/null || true) - if [ -n "$mc_endpoint" ]; then break; fi - sleep 2 -done -mc_want_endpoint="$MOONCAKE_CB_NAME.$MOONCAKE_SMOKE_NS.svc.cluster.local:50051" -if [ "$mc_endpoint" != "$mc_want_endpoint" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get cb "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake status.endpoint=$mc_endpoint, want $mc_want_endpoint (master RPC host:port via ResolveCacheServer)" -fi -log "Mooncake status.endpoint=$mc_endpoint" - -# The rendered Service must expose the RPC port (50051) FIRST — serviceEndpoint -# publishes Ports[0], and the engine wire dials it via mooncakestore://. -mc_svc_port="$(kubectl -n "$MOONCAKE_SMOKE_NS" get svc "$MOONCAKE_CB_NAME" \ - -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || true)" -if [ "$mc_svc_port" != "50051" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get svc "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake Service first port=$mc_svc_port, want 50051" -fi - -# Mooncake's transfer engine is a peer-to-peer mesh: the master hands back a -# directory pointer and the engine then dials a real node IP on a dynamically -# negotiated port. Two provisioning properties make that reachable, and BOTH are -# asserted here against the real install because losing either is silent — the -# backend still reconciles and reports an endpoint while transferring zero KV. -# -# - headless Service (clusterIP: None): a virtual ClusterIP forwards only the -# ports declared above and strands the dynamic ones. -# - hostNetwork master pod: CNI overlay pod IPs are not reachable for the mesh. -mc_svc_clusterip="$(kubectl -n "$MOONCAKE_SMOKE_NS" get svc "$MOONCAKE_CB_NAME" \ - -o jsonpath='{.spec.clusterIP}' 2>/dev/null || true)" -if [ "$mc_svc_clusterip" != "None" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get svc "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake Service clusterIP=$mc_svc_clusterip, want None (headless; a virtual IP strands the mesh's dynamic ports)" -fi - -mc_host_network="$(kubectl -n "$MOONCAKE_SMOKE_NS" get deploy "$MOONCAKE_CB_NAME" \ - -o jsonpath='{.spec.template.spec.hostNetwork}' 2>/dev/null || true)" -if [ "$mc_host_network" != "true" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get deploy "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake master hostNetwork=$mc_host_network, want true (overlay pod IPs are unreachable for the transfer-engine mesh)" -fi - -# A hostNetwork pod binds the node's ports, so a rolling surge would collide. -mc_strategy="$(kubectl -n "$MOONCAKE_SMOKE_NS" get deploy "$MOONCAKE_CB_NAME" \ - -o jsonpath='{.spec.strategy.type}' 2>/dev/null || true)" -if [ "$mc_strategy" != "Recreate" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get deploy "$MOONCAKE_CB_NAME" -o yaml || true - fail "Mooncake master rollout strategy=$mc_strategy, want Recreate (hostNetwork pods collide on the node's ports)" -fi -log "Mooncake master: hostNetwork=true, Recreate strategy, headless Service" - -# The managed Deployment must reach Available: the stand-in master accepts TCP -# on 50051 so the controller-rendered readiness probe passes — proving -# ResolveCacheServer rendered a workload that actually comes up under a real -# install. (CacheBackend Ready is deliberately NOT asserted: no engine is wired -# here, so the KV-event readiness gate legitimately holds it at -# AwaitingFirstKVEvent — orthogonal to the managed-reconcile contract.) -log "waiting up to ${READY_TIMEOUT} for the Mooncake master Deployment to be Available" -if ! kubectl -n "$MOONCAKE_SMOKE_NS" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ - deployment/"$MOONCAKE_CB_NAME" >/dev/null 2>&1; then - kubectl -n "$MOONCAKE_SMOKE_NS" get deploy "$MOONCAKE_CB_NAME" -o yaml || true - kubectl -n "$MOONCAKE_SMOKE_NS" get pods -o wide || true - fail "Mooncake master Deployment did not reach Available within ${READY_TIMEOUT}" -fi -log "Mooncake master Deployment Available; canonical managed Mooncake reconcile verified end-to-end" - -# The operator-facing half of the Mooncake data plane: spec.integration.engineHostNetwork -# moves matched ENGINE pods onto the host network. Assert it against the real -# installed webhook, not just the adapter unit test — this is the surface the -# operator's own pod crosses, and getting it wrong is silent (Ready, zero KV). -# -# `pause` with the conventional `vllm` container name, mirroring the External -# engine-pod fixture above: we inspect the admitted .spec, so the pod need not -# run. No containerPort is declared — under hostNetwork the API server defaults -# hostPort=containerPort, and a bound node port would make this pod unschedulable -# for reasons unrelated to what is being asserted. -MOONCAKE_ENGINE_POD="mooncake-engine" -cat </dev/null || fail "kubectl apply Mooncake engine pod failed" -apiVersion: v1 -kind: Pod -metadata: - name: $MOONCAKE_ENGINE_POD - namespace: $MOONCAKE_SMOKE_NS - labels: - app.kubernetes.io/name: vllm -spec: - containers: - - name: vllm - image: registry.k8s.io/pause:3.10 -EOF - -mc_pod_hostnet="$(kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" \ - -o jsonpath='{.spec.hostNetwork}' 2>/dev/null || true)" -if [ "$mc_pod_hostnet" != "true" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" -o yaml || true - fail "Mooncake engine pod hostNetwork=$mc_pod_hostnet, want true (spec.integration.engineHostNetwork opt-in was not applied; the engine cannot reach the mesh)" -fi - -# Without ClusterFirstWithHostNet a hostNetwork pod inherits the node's resolver -# and cannot resolve status.endpoint, which is a Service DNS name. -mc_pod_dns="$(kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" \ - -o jsonpath='{.spec.dnsPolicy}' 2>/dev/null || true)" -if [ "$mc_pod_dns" != "ClusterFirstWithHostNet" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" -o yaml || true - fail "Mooncake engine pod dnsPolicy=$mc_pod_dns, want ClusterFirstWithHostNet (the master endpoint is a Service DNS name)" -fi - -# hostNetwork travels WITH the connector, never alone: assert the pod is actually -# wired to the master, so a regression that grants the privilege while dropping -# the wiring (or vice versa) fails here. -mc_pod_url="$(kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" \ - -o jsonpath='{.spec.containers[?(@.name=="vllm")].env[?(@.name=="LMCACHE_REMOTE_URL")].value}' 2>/dev/null || true)" -if [ "$mc_pod_url" != "mooncakestore://$mc_want_endpoint" ]; then - kubectl -n "$MOONCAKE_SMOKE_NS" get pod "$MOONCAKE_ENGINE_POD" -o yaml || true - fail "Mooncake engine pod LMCACHE_REMOTE_URL=$mc_pod_url, want mooncakestore://$mc_want_endpoint" -fi -log "Mooncake engine pod: hostNetwork=true, dnsPolicy=ClusterFirstWithHostNet, wired to mooncakestore://$mc_want_endpoint" +case "$doctor_rc" in 0|1|2) ;; *) fail "doctor exited with unexpected code $doctor_rc" ;; esac +grep -Fq '"summary"' "$LOG_DIR/doctor.json" || fail "doctor JSON summary is missing" +grep -Fq '"findings"' "$LOG_DIR/doctor.json" || fail "doctor JSON findings are missing" -kubectl delete namespace "$MOONCAKE_SMOKE_NS" --ignore-not-found --wait=false >/dev/null 2>&1 || true +log "re-applying the bundle as an idempotent upgrade check" +kubectl apply -k "$tmpdir/config/default" >/dev/null +kubectl -n "$SYSTEM_NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ + deployment/inference-cache-controller-manager deployment/inference-cache-server +kubectl -n "$SMOKE_NAMESPACE" get cachebackend host-only managed-redis >/dev/null -log "PASS — install bundle came up, CacheIndex + CacheTenant status writing, PromptTemplate + PDTopology schema-only surfaces, server HTTP surface, CachePolicy push adoption, gRPC fail-open (plaintext default), adapter (LoRA) index partitioning on LookupRoute, CacheBackend ↔ engine-pod binding signals + drift cadence, provider resource defaults + thread-through, External backend end-to-end, Events-only + native SGLang HiCache engine-local lifecycles, /snapshot + /policy + /probe unauth rejection, audience binding on all three endpoints, the opt-in gRPC TLS overlay (incl. the existing LookupRoute call pattern over TLS), kernel-check injection shape + report-only FAIL condition path (EngineKernelsHealthy=False/KernelLoadFailed), the operator 'inferencecache doctor' CLI against the live install, the managed Mooncake backend provisioning contract (stand-in master reaches Available on hostNetwork behind a headless Service, Recreate strategy, mooncakestore:// RPC endpoint in status) plus the engineHostNetwork opt-in end-to-end (warning fires only without it; a matched engine pod is admitted onto hostNetwork with ClusterFirstWithHostNet and the mooncakestore:// connector, while a non-Mooncake engine pod stays on the pod network; real engine KV transfer is NOT exercised here), and every config/samples/ manifest applies cleanly — all work" +log "PASS: default install, control-plane APIs, server surfaces, samples, doctor, typed MP admission, managed Redis, and idempotent re-apply" diff --git a/docs/reference-stack/scripts/phase5_upgrade_smoke.sh b/docs/reference-stack/scripts/phase5_upgrade_smoke.sh new file mode 100755 index 00000000..95fd5ffa --- /dev/null +++ b/docs/reference-stack/scripts/phase5_upgrade_smoke.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Installs the last Phase 5 revision, creates only Phase 5 typed PodLocal MP +# objects, then upgrades the CRD and controller to the current checkout. This +# intentionally does not create or migrate legacy IP objects: the recorded +# Phase 0/5 consumer audit found no supported population for them. + +set -euo pipefail + +PHASE5_COMMIT="${PHASE5_COMMIT:-10178558bfca308ee3a4b0d584efe4ed3b91197d}" +PHASE5_TAG="${PHASE5_TAG:-phase5-upgrade-base}" +TAG="${TAG:-${GITHUB_SHA:-$(git rev-parse HEAD)}}" +REGISTRY="${REGISTRY:-ghcr.io/cachebox-project}" +KIND_CLUSTER="${KIND_CLUSTER:-ic-phase5-upgrade}" +SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-inference-cache-system}" +SMOKE_NAMESPACE="${SMOKE_NAMESPACE:-ic-phase5-upgrade}" +CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.1}" +READY_TIMEOUT="${READY_TIMEOUT:-180s}" +KEEP_CLUSTER="${KEEP_CLUSTER:-0}" +LOG_DIR="${LOG_DIR:-/tmp/phase5-upgrade-smoke-logs}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +CURRENT_CONTROLLER_IMG="$REGISTRY/inference-cache-controller:$TAG" +CURRENT_SERVER_IMG="$REGISTRY/inference-cache-server:$TAG" +PHASE5_CONTROLLER_IMG="$REGISTRY/inference-cache-controller:$PHASE5_TAG" +PHASE5_SERVER_IMG="$REGISTRY/inference-cache-server:$PHASE5_TAG" +KIND="${KIND:-$REPO_ROOT/bin/kind}" +[ -x "$KIND" ] || KIND=kind + +log() { printf '[phase5-upgrade-smoke] %s\n' "$*"; } +fail() { printf '[phase5-upgrade-smoke] ERROR: %s\n' "$*" >&2; exit 1; } + +for binary in docker git kubectl tar "$KIND"; do + command -v "$binary" >/dev/null 2>&1 || fail "missing required tool: $binary" +done +git -C "$REPO_ROOT" cat-file -e "$PHASE5_COMMIT^{commit}" \ + || fail "Phase 5 commit is unavailable: $PHASE5_COMMIT" + +mkdir -p "$LOG_DIR" +tmpdir="$(mktemp -d)" +created_cluster=0 + +collect_diagnostics() { + kubectl get cachebackends -A -o yaml >"$LOG_DIR/cachebackends.yaml" 2>&1 || true + kubectl -n "$SYSTEM_NAMESPACE" get all -o wide >"$LOG_DIR/system.txt" 2>&1 || true + kubectl -n "$SYSTEM_NAMESPACE" logs deployment/inference-cache-controller-manager --all-containers >"$LOG_DIR/controller.log" 2>&1 || true +} + +cleanup() { + rm -rf "$tmpdir" + if [ "$created_cluster" = "1" ] && [ "$KEEP_CLUSTER" != "1" ]; then + "$KIND" delete cluster --name "$KIND_CLUSTER" >/dev/null 2>&1 || true + fi +} + +on_exit() { + rc=$? + [ "$rc" -eq 0 ] || collect_diagnostics + cleanup + exit "$rc" +} +trap on_exit EXIT + +render_config() { + local source_dir="$1" destination="$2" controller_image="$3" server_image="$4" + cp -R "$source_dir/config" "$destination" + sed -i.bak \ + -e "/^- name: controller$/,/^- name: server$/ { s|^ newName: .*| newName: ${controller_image%:*}|; s|^ newTag: .*| newTag: ${controller_image##*:}|; }" \ + -e "/^- name: server$/,$ { s|^ newName: .*| newName: ${server_image%:*}|; s|^ newTag: .*| newTag: ${server_image##*:}|; }" \ + "$destination/default/kustomization.yaml" + rm -f "$destination/default/kustomization.yaml.bak" +} + +if "$KIND" get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER"; then + log "reusing kind cluster $KIND_CLUSTER" +else + "$KIND" create cluster --name "$KIND_CLUSTER" --wait 120s + created_cluster=1 +fi +kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null + +log "installing cert-manager $CERT_MANAGER_VERSION" +kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" >/dev/null +kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout=180s + +phase5_src="$tmpdir/phase5-src" +mkdir -p "$phase5_src" +git -C "$REPO_ROOT" archive "$PHASE5_COMMIT" | tar -x -C "$phase5_src" + +log "building and installing Phase 5 at $PHASE5_COMMIT" +make -C "$phase5_src" image-build TAG="$PHASE5_TAG" REGISTRY="$REGISTRY" +"$KIND" load docker-image "$PHASE5_CONTROLLER_IMG" --name "$KIND_CLUSTER" +"$KIND" load docker-image "$PHASE5_SERVER_IMG" --name "$KIND_CLUSTER" +render_config "$phase5_src" "$tmpdir/phase5-config" "$PHASE5_CONTROLLER_IMG" "$PHASE5_SERVER_IMG" +kubectl apply -k "$tmpdir/phase5-config/default" >/dev/null +kubectl -n "$SYSTEM_NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ + deployment/inference-cache-controller-manager deployment/inference-cache-server + +kubectl create namespace "$SMOKE_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +log "creating Phase 5 typed MP objects" +cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply -f - >/dev/null +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: phase5-host-only +spec: + runtime: VLLM + type: LMCache + engineSelector: + matchLabels: + app: phase5-engine + integration: + role: ReadWrite + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 1Gi + maxWorkers: 1 + resources: + requests: {cpu: "1", memory: 2Gi} + limits: {memory: 2Gi} +--- +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: phase5-managed-redis +spec: + runtime: SGLang + type: LMCache + engineSelector: + matchLabels: + app: phase5-sglang + integration: + role: ReadWrite + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 1Gi + maxWorkers: 1 + resources: + requests: {cpu: "1", memory: 2Gi} + limits: {memory: 2Gi} + remoteStorage: + provider: Redis + ownership: Managed + redis: {} +EOF + +host_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-host-only -o jsonpath='{.metadata.uid}')" +redis_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.metadata.uid}')" +[ -n "$host_uid" ] && [ -n "$redis_uid" ] || fail "Phase 5 objects were not persisted" + +log "upgrading CRDs and workloads to the current Phase 7 checkout" +make -C "$REPO_ROOT" image-build TAG="$TAG" REGISTRY="$REGISTRY" +"$KIND" load docker-image "$CURRENT_CONTROLLER_IMG" --name "$KIND_CLUSTER" +"$KIND" load docker-image "$CURRENT_SERVER_IMG" --name "$KIND_CLUSTER" +render_config "$REPO_ROOT" "$tmpdir/current-config" "$CURRENT_CONTROLLER_IMG" "$CURRENT_SERVER_IMG" +kubectl apply -k "$tmpdir/current-config/default" >/dev/null +kubectl -n "$SYSTEM_NAMESPACE" rollout status deployment/inference-cache-controller-manager --timeout="$READY_TIMEOUT" +kubectl -n "$SYSTEM_NAMESPACE" rollout status deployment/inference-cache-server --timeout="$READY_TIMEOUT" + +[ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-host-only -o jsonpath='{.metadata.uid}')" = "$host_uid" ] \ + || fail "host-only Phase 5 object was replaced during upgrade" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.metadata.uid}')" = "$redis_uid" ] \ + || fail "managed-Redis Phase 5 object was replaced during upgrade" +for backend in phase5-host-only phase5-managed-redis; do + [ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend "$backend" -o jsonpath='{.spec.lmCache.topology}')" = "PodLocal" ] \ + || fail "$backend lost its typed MP topology" +done + +for _ in $(seq 1 60); do + endpoint="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.status.remoteStorage.endpoint}' 2>/dev/null || true)" + [ "$endpoint" = "phase5-managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] && break + sleep 1 +done +[ "${endpoint:-}" = "phase5-managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] \ + || fail "managed Redis did not reconcile after upgrade" + +pod_json="$tmpdir/upgraded-admission.json" +cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -o json -f - >"$pod_json" +apiVersion: v1 +kind: Pod +metadata: + name: phase5-engine + labels: + app: phase5-engine +spec: + containers: + - name: vllm + image: busybox:1.36 + command: ["sh", "-c", "sleep 3600"] +EOF +grep -Fq 'lmcache-mp-server' "$pod_json" || fail "upgraded admission did not inject the MP server" +grep -Fq 'LMCacheMPConnector' "$pod_json" || fail "upgraded admission did not inject the MP connector" + +log "PASS: Phase 5 typed objects persisted and reconciled through the Phase 7 upgrade" diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 9e5104e8..a131b64b 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -17,8 +17,7 @@ silently. the prefix from the `metricNamespace` constant in [`internal/server/metrics.go`](../../internal/server/metrics.go); controller- binary metrics declare it inline on each `prometheus.NewXVec` - declaration in `internal/controller/` (see - `backendServerRestartCascadesTotal` for the pattern) — the two + declaration in `internal/controller/` — the two processes use separate Prometheus registries, so no shared package-level constant is used to enforce the prefix today. Anything not matching that prefix is from a standard collector @@ -84,7 +83,6 @@ Emitted by the `cmd/controller` binary, registered into the controller-runtime m | Metric | Labels | Meaning | Notes | |---|---|---|---| | `inferencecache_backend_probe_result_total` | `backend`, `stage`, `result` | One increment per stage per probe call the CacheBackend reconciler issues to the server's `/probe` endpoint. `backend` is the canonical `/`; `stage` ∈ `ingest` / `routing` / `t2`; `result` ∈ `ok` / `failed` / `skipped`. A successful probe call emits three increments (one per stage); an HTTP-level failure to reach `/probe` emits zero (no per-stage outcome was observed — the call itself failed). Skipped stages count too — the metric reflects "what the probe round-trip looked like," not "what was exercised." | `backend × stage × result` cardinality is bounded by the size of the CacheBackend fleet × 3 × 3, comfortably small. Probe rate (~once per backend per 30s) keeps total emission tame. Dashboards key off the `result="failed"` slice for the alerting signal — a steady rate means the cache plane has a known regression for that backend; the `stage` label points at which layer broke. See [`docs/design/cachebackend-api.md#functional-probe-gate`](../design/cachebackend-api.md#functional-probe-gate) for the semantics. The `ServerProbeFail` alert in `config/observability/{alerting-rules,prometheus-rules}.yaml` is wired off this metric. | -| `inferencecache_backend_server_restart_cascades_total` | `namespace`, `backend`, `reason` | One increment per cascade-restart **decision** the `CacheBackend` reconciler emits when it observes a cache-server-pod replacement that warrants engine recovery. **The counter advances per cascade EVENT, not per Deployment patched** — a cascade that matches zero injected engine `Deployment`s today still counts as one event (the controller decided recovery was needed; the engine fleet may simply not be deployed yet, or `spec.engineSelector` is being rewired). The decision fires after the rate-limit window has elapsed and after the engine-Deployment annotates succeed — BEFORE the subsequent `status.observedServerInstance` patch. The metric reflects the cascade decision the moment it commits — any matched engine `Deployment`s have already been annotated and the rollout that drives the recovery is in flight — rather than lagging behind a transient status-write failure. A zero-match cascade (no engines injected yet, or `spec.engineSelector` is being rewired) still increments because the controller's "decided to recover" state is operator-actionable even when no engine rolled. Double-counting on retry is prevented by an in-process `(key, currentID)` ledger: a subsequent reconcile that re-enters the cascade branch with the same identifier does not advance the counter. | NOT a raw restart count: the cascade is rate-limited to at most once per ~30s per backend (see `DefaultMinServerRestartCascadeInterval`), so a crash-looping cache-server that restarts 10× inside one window still increments this counter once. For raw cache-server pod restart rate, scrape `kube_pod_container_status_restarts_total` from kube-state-metrics instead. Today `reason` is always `server_instance_changed`; future operator-initiated "force cascade" surfaces would add their own value. A series is created lazily on the first cascade — a backend that never cascades emits nothing. The cascade itself is the operator-side recovery for the upstream LMCache `LMServerConnector` EPIPE-on-restart bug ([LMCache/LMCache#3565](https://github.com/LMCache/LMCache/issues/3565)); see [`docs/design/cachebackend-api.md` `observedServerInstance`](../design/cachebackend-api.md). | | `inferencecache_backend_t2_query_tokens_total` | `backend` | **Monotonic** count of tier-2 (external offload) query tokens observed per CacheBackend — the **activity signal** for tier-2-degradation alerting: `rate(...)` separates an actively-queried backend (degraded when its hit-rate is `0`) from one that took a few cold misses and went idle. The CacheIndex poller accumulates only the **positive per-tick deltas** of the per-backend aggregate cumulative, so a drop from replica/tenant churn or an engine restart is clamped out and `rate()` never sees a phantom reset. | `backend` is the canonical `/` (Prometheus injects the install `namespace`); same present-when-exercised lifecycle + stale-series pruning as `inferencecache_backend_t2_hit_rate`. Intended for `rate(...) > 1000` tokens/sec activity gating in a tier-2-degradation alert (shipped separately in the observability bundle). | --- @@ -140,12 +138,6 @@ with OTEL collectors) without bumping `v1alpha1`. registered into the controller-runtime metrics registry on `init()`. This is a separate `prometheus.Registry` from the server binary's per- Service registry. -- **`backendServerRestartCascadesTotal` writer:** the `CacheBackend` - reconciler increments it once per cascade in - [`internal/controller/cachebackend_server_restart.go`](../../internal/controller/cachebackend_server_restart.go). - See the `reconcileServerInstance` godoc for when a cascade is and is - not emitted (rate-limit, strict-superset midpoints, converged - scale-ups, stale-while-unavailable). - **`inferencecache_backend_probe_result_total` writer:** the `CacheBackend` reconciler in [`internal/controller/cachebackend_probe.go`](../../internal/controller/cachebackend_probe.go) @@ -266,9 +258,7 @@ Two binaries each expose their own `/metrics` endpoint — separate processes, s keeping the surface narrow makes it test-mockable. - **Controller binary (`cmd/controller`)**: declare a package-level `prometheus.NewCounterVec` / `NewGaugeVec` / etc. var in the - reconciler / webhook file that uses it (e.g. - `backendServerRestartCascadesTotal` in - `internal/controller/cachebackend_server_restart.go`); register it + reconciler / webhook file that uses it; register it into `sigs.k8s.io/controller-runtime/pkg/metrics.Registry` from an `init()` so it appears on the manager's `/metrics` endpoint without a separate plumbing path. Add a package-private @@ -284,8 +274,7 @@ Two binaries each expose their own `/metrics` endpoint — separate processes, s 4. **Wire test coverage.** Server-binary metrics: add an assertion in `internal/server/metrics_test.go`. Controller-binary metrics: add an assertion in a `_test.go` file alongside the reconciler that increments - them (e.g. `cachebackend_server_restart_test.go` — see the - `cascadeRestartsCount` helper for the pattern). In both cases verify + them. In both cases verify the metric appears in `/metrics` output with the expected name and labels. 5. **Flag the schema impact in the PR description.** If the metric is a diff --git a/internal/adapters/builtin/registry.go b/internal/adapters/builtin/registry.go index db77b842..6b0cfb81 100644 --- a/internal/adapters/builtin/registry.go +++ b/internal/adapters/builtin/registry.go @@ -22,7 +22,6 @@ type Registries struct { // It belongs to the built-in composition rather than the public adapter seam. type Options struct { SubscriberImage string - LMCacheServerImage string PolicyServerGRPCAddress string } @@ -35,14 +34,11 @@ func New(opts Options) Registries { } runtimeRegistry := adapterruntime.NewRegistry() runtimeRegistry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(subscriber)) - runtimeRegistry.Register(builtinruntime.NewVLLMLMCacheAdapter(subscriber)) runtimeRegistry.Register(builtinruntime.NewSGLangLMCacheAdapter(subscriber)) runtimeRegistry.Register(builtinruntime.NewSGLangHiCacheAdapter(subscriber)) return Registries{ Runtime: runtimeRegistry, - Storage: builtinstorage.DefaultRegistry( - builtinstorage.WithLMCacheServerImage(opts.LMCacheServerImage), - ), + Storage: builtinstorage.DefaultRegistry(), } } diff --git a/internal/adapters/builtin/registry_test.go b/internal/adapters/builtin/registry_test.go index 15073274..71f8fed8 100644 --- a/internal/adapters/builtin/registry_test.go +++ b/internal/adapters/builtin/registry_test.go @@ -11,109 +11,23 @@ import ( adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" ) -func TestNewIncludesEveryShippingRuntimeAdapter(t *testing.T) { - t.Parallel() - - registry := New(Options{}).Runtime - for _, tc := range []struct { - name string - runtime adapterruntime.RuntimeID - backend cachev1alpha1.CacheBackendType - integration *cachev1alpha1.CacheBackendIntegrationSpec - }{ - {name: "vllm lmcache", runtime: adapterruntime.RuntimeVLLM, backend: cachev1alpha1.CacheBackendTypeLMCache}, - {name: "sglang lmcache", runtime: adapterruntime.RuntimeSGLang, backend: cachev1alpha1.CacheBackendTypeLMCache}, - {name: "sglang hicache", runtime: adapterruntime.RuntimeSGLang, backend: cachev1alpha1.CacheBackendTypeSGLangHiCache}, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - cache := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Type: tc.backend, Integration: tc.integration, - }} - if _, err := registry.Select(tc.runtime, cache); err != nil { - t.Fatalf("Select(%q, %q): %v", tc.runtime, tc.backend, err) - } - }) - } -} - -func TestNewSelectsTypedVLLMMPBeforeLegacyAdapter(t *testing.T) { - t.Parallel() - - registry := New(Options{}).Runtime - typed := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - LMCache: &cachev1alpha1.LMCacheEngineSpec{Topology: cachev1alpha1.LMCacheTopologyPodLocal}, - }} - adapter, err := registry.Select(adapterruntime.RuntimeVLLM, typed) - if err != nil { - t.Fatalf("Select typed vLLM adapter: %v", err) +func TestNewRegistersCurrentRuntimeAndStorageAdapters(t *testing.T) { + registries := New(Options{}) + if got := registries.Runtime.Len(); got != 3 { + t.Fatalf("runtime registry length = %d, want 3", got) } - if _, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter); !ok { - t.Fatalf("typed vLLM adapter = %T, want LMCacheMPRuntimeAdapter", adapter) - } - - legacy := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }} - adapter, err = registry.Select(adapterruntime.RuntimeVLLM, legacy) - if err != nil { - t.Fatalf("Select legacy vLLM adapter: %v", err) - } - if _, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter); ok { - t.Fatalf("legacy vLLM adapter = %T, unexpectedly implements LMCacheMPRuntimeAdapter", adapter) - } -} - -func TestNewIncludesShippingStorageProviders(t *testing.T) { - t.Parallel() - - registry := New(Options{}).Storage - for _, provider := range []cachev1alpha1.CacheBackendRemoteStorageProvider{ - cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - } { - for _, ownership := range []cachev1alpha1.CacheBackendRemoteStorageOwnership{ - cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - } { - storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: provider, Ownership: ownership, - } - if _, err := registry.Select(storage); err != nil { - t.Fatalf("Select(%q, %q): %v", provider, ownership, err) - } - } - } -} - -func TestNewPassesLMCacheServerImageToStorageRegistry(t *testing.T) { - t.Parallel() - - const image = "registry.example/lmcache:operator-default" cache := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, + LMCache: &cachev1alpha1.LMCacheEngineSpec{Topology: cachev1alpha1.LMCacheTopologyPodLocal}, }} - - registry := New(Options{LMCacheServerImage: image}).Storage - provider, err := registry.Select(cache.Spec.RemoteStorage) - if err != nil { - t.Fatalf("Select: %v", err) - } - rendered, err := provider.Render(cache) - if err != nil { - t.Fatalf("Render: %v", err) - } - if got := rendered.PodSpec.Containers[0].Image; got != image { - t.Fatalf("container image = %q, want %q", got, image) + if _, err := registries.Runtime.Select(adapterruntime.RuntimeVLLM, cache); err != nil { + t.Fatalf("select vLLM LMCache MP adapter: %v", err) + } + if _, err := registries.Storage.Select(&cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + }); err != nil { + t.Fatalf("select managed Redis provider: %v", err) } } diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go index 61a04db6..2b9d8397 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go @@ -89,10 +89,6 @@ func renderLMCachePodLocalServer(pod *corev1.PodSpec, engineContainerName string engine := &work.Containers[engineIndex] owned := lmCacheMPWireIsOurs(pod) - if legacy := findContainerByName(work.InitContainers, sglangMPWorkerContainerName); legacy != nil { - return "", fmt.Errorf("render LMCache MP server: pod already has legacy native sidecar %q; remove the legacy topology-less injection before enabling typed PodLocal", legacy.Name) - } - if cfg.WriteClientConfig { if existing := mountAtPath(engine.VolumeMounts, lmCacheMPConfigMountPath); existing != nil && !(owned && existing.Name == lmCacheMPConfigVolumeName) { diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go index f4d99a19..8aa0e989 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go @@ -16,11 +16,11 @@ import ( backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" ) -const testLMCacheServerImage = "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const testMPServerImage = "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" func testLMCacheMPConfig() lmCacheMPServerConfig { return lmCacheMPServerConfig{ - Image: testLMCacheServerImage, + Image: testMPServerImage, Port: 6500, ChunkSizeTokens: 256, L1Capacity: resource.MustParse("4Gi"), @@ -69,7 +69,7 @@ func TestRenderLMCachePodLocalServerGolden(t *testing.T) { if server == nil { t.Fatalf("MP server missing: %+v", pod.InitContainers) } - if server.Image != testLMCacheServerImage || server.Image == pod.Containers[0].Image { + if server.Image != testMPServerImage || server.Image == pod.Containers[0].Image { t.Fatalf("server image = %q, engine image = %q", server.Image, pod.Containers[0].Image) } if server.RestartPolicy == nil || *server.RestartPolicy != corev1.ContainerRestartPolicyAlways { diff --git a/internal/adapters/builtin/runtime/lmcachecheck.go b/internal/adapters/builtin/runtime/lmcachecheck.go index 24c39dfb..b7edf110 100644 --- a/internal/adapters/builtin/runtime/lmcachecheck.go +++ b/internal/adapters/builtin/runtime/lmcachecheck.go @@ -164,7 +164,7 @@ func kernelCheckResources() corev1.ResourceRequirements { // a vLLM+LMCache engine pod, or nil when the configured gate does not apply. // Auto mode checks GPU pods in report-only mode; report-only and strict force // injection; off disables it. -func (vllmLMCacheAdapter) KernelCheckInitContainer(cache *cachev1alpha1.CacheBackend, pod *corev1.Pod) (*corev1.Container, error) { +func (vllmLMCacheMPAdapter) KernelCheckInitContainer(cache *cachev1alpha1.CacheBackend, pod *corev1.Pod) (*corev1.Container, error) { if cache == nil || pod == nil { return nil, nil } diff --git a/internal/adapters/builtin/runtime/lmcachecheck_test.go b/internal/adapters/builtin/runtime/lmcachecheck_test.go index 83b64458..5229ef1e 100644 --- a/internal/adapters/builtin/runtime/lmcachecheck_test.go +++ b/internal/adapters/builtin/runtime/lmcachecheck_test.go @@ -40,7 +40,7 @@ func cbWithKernelCheck(mode string) *cachev1alpha1.CacheBackend { } func TestKernelCheckAutoInjectsOnGPUPod(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) c, err := a.KernelCheckInitContainer(cbWithKernelCheck(""), gpuEnginePod("vllm/img:cu129")) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -78,7 +78,7 @@ func TestKernelCheckAutoInjectsOnGPUPod(t *testing.T) { } func TestKernelCheckCommandIdenticalAcrossModesEnvDiffers(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) ro, _ := a.KernelCheckInitContainer(cbWithKernelCheck(enginebinding.KernelCheckModeReportOnly), gpuEnginePod("img")) st, _ := a.KernelCheckInitContainer(cbWithKernelCheck(enginebinding.KernelCheckModeStrict), gpuEnginePod("img")) if ro == nil || st == nil { @@ -128,7 +128,7 @@ func strictEnvValue(c *corev1.Container) (string, int) { } func TestKernelCheckStripsInheritedStrictEnv(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) pod := gpuEnginePod("img") // The engine container carries a stray KERNEL_CHECK_STRICT=1. It must NOT // leak into the report-only check (which would turn it fail-closed) and must @@ -147,7 +147,7 @@ func TestKernelCheckStripsInheritedStrictEnv(t *testing.T) { } func TestKernelCheckAutoSkipsCPUPod(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) cpuPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ Name: EngineContainerName, Image: "vllm/cpu", }}}} @@ -161,7 +161,7 @@ func TestKernelCheckAutoSkipsCPUPod(t *testing.T) { } func TestKernelCheckOffSkipsEvenGPU(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) c, _ := a.KernelCheckInitContainer(cbWithKernelCheck(enginebinding.KernelCheckModeOff), gpuEnginePod("img")) if c != nil { t.Fatal("off mode must never inject") @@ -169,7 +169,7 @@ func TestKernelCheckOffSkipsEvenGPU(t *testing.T) { } func TestKernelCheckReportOnlyInjectsOnCPU(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) cpuPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName, Image: "img"}}}} c, _ := a.KernelCheckInitContainer(cbWithKernelCheck(enginebinding.KernelCheckModeReportOnly), cpuPod) if c == nil { @@ -183,7 +183,7 @@ func TestKernelCheckReportOnlyInjectsOnCPU(t *testing.T) { } func TestKernelCheckStrictSetsStrictEnv(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) c, _ := a.KernelCheckInitContainer(cbWithKernelCheck(enginebinding.KernelCheckModeStrict), gpuEnginePod("img")) if c == nil { t.Fatal("strict must inject") @@ -200,7 +200,7 @@ func TestKernelCheckStrictSetsStrictEnv(t *testing.T) { } func TestKernelCheckMultiContainerNoEngineNameSkips(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ {Name: "foo", Image: "a", Resources: corev1.ResourceRequirements{Limits: corev1.ResourceList{gpuResourceName: resource.MustParse("1")}}}, {Name: "bar", Image: "b"}, @@ -215,7 +215,7 @@ func TestKernelCheckMultiContainerNoEngineNameSkips(t *testing.T) { } func TestKernelCheckCopiesEngineEnvironment(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) + a := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(enginebinding.InitContainerProvider) nonRoot := true engineSC := &corev1.SecurityContext{RunAsNonRoot: &nonRoot} pod := gpuEnginePod("img") diff --git a/internal/adapters/builtin/runtime/runtime_helpers.go b/internal/adapters/builtin/runtime/runtime_helpers.go new file mode 100644 index 00000000..52c6ee8e --- /dev/null +++ b/internal/adapters/builtin/runtime/runtime_helpers.go @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) + +const ( + EngineContainerName = "vllm" + EnvInferenceCacheFailOpen = "INFERENCECACHE_FAIL_OPEN" + EnvPythonHashSeed = "PYTHONHASHSEED" + defaultPythonHashSeed = "0" + defaultEngineKVTransferConfigArg = "--kv-transfer-config" +) + +func UpsertFlag(args []string, flag string) []string { + for _, arg := range args { + if arg == flag { + return args + } + } + return append(args, flag) +} + +func validateInjectPodCacheInputs(pod *corev1.PodSpec, cache *cachev1alpha1.CacheBackend, role string) error { + if pod == nil { + return fmt.Errorf("inject %s config: pod is nil", role) + } + if cache == nil { + return fmt.Errorf("inject %s config: cache is nil", role) + } + if len(pod.Containers) == 0 { + return fmt.Errorf("inject %s config: pod has no containers", role) + } + return nil +} + +func EngineContainerIndexNamed(pod *corev1.PodSpec, name string) (int, error) { + for i := range pod.Containers { + if pod.Containers[i].Name == name { + return i, nil + } + } + if len(pod.Containers) == 1 { + return 0, nil + } + names := make([]string, len(pod.Containers)) + for i := range pod.Containers { + names[i] = pod.Containers[i].Name + } + return -1, fmt.Errorf("inject engine config: pod has %d containers %v but none is named %q; injecting engine flags into unrelated sidecars would crash them — name the engine container %q", + len(pod.Containers), names, name, name) +} + +func FailOpenString(cache *cachev1alpha1.CacheBackend) string { + if cachev1alpha1.IntegrationFailOpen(cache.Spec.Integration) { + return "true" + } + return "false" +} + +func IntegrationRole(cache *cachev1alpha1.CacheBackend) cachev1alpha1.CacheBackendIntegrationRole { + if cache.Spec.Integration == nil || cache.Spec.Integration.Role == "" { + return cachev1alpha1.CacheBackendIntegrationRoleReadWrite + } + return cache.Spec.Integration.Role +} + +func UpsertArgPair(args []string, flag, value string) []string { + prefix := flag + "=" + for i, arg := range args { + switch { + case arg == flag: + if i+1 < len(args) { + args[i+1] = value + return args + } + return append(args, value) + case strings.HasPrefix(arg, prefix): + args[i] = flag + out := make([]string, 0, len(args)+1) + out = append(out, args[:i+1]...) + out = append(out, value) + out = append(out, args[i+1:]...) + return out + } + } + return append(args, flag, value) +} + +func UpsertEnv(env []corev1.EnvVar, want corev1.EnvVar) []corev1.EnvVar { + for i := range env { + if env[i].Name == want.Name { + env[i].Value = want.Value + env[i].ValueFrom = want.ValueFrom + return env + } + } + return append(env, want) +} + +func removeEnv(env []corev1.EnvVar, name string) []corev1.EnvVar { + out := env[:0] + for _, entry := range env { + if entry.Name != name { + out = append(out, entry) + } + } + return out +} + +func mountAtPath(mounts []corev1.VolumeMount, path string) *corev1.VolumeMount { + for i := range mounts { + if mounts[i].MountPath == path { + return &mounts[i] + } + } + return nil +} + +func lmCacheMPServerSecurityContext(engine *corev1.SecurityContext) *corev1.SecurityContext { + no := false + securityContext := &corev1.SecurityContext{ + AllowPrivilegeEscalation: &no, + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } + if engine != nil { + securityContext.RunAsNonRoot = engine.RunAsNonRoot + securityContext.RunAsUser = engine.RunAsUser + securityContext.RunAsGroup = engine.RunAsGroup + } + return securityContext +} + +func adoptContainer(containers []corev1.Container, want corev1.Container, owned bool) ([]corev1.Container, error) { + for i := range containers { + if containers[i].Name != want.Name { + continue + } + if !owned { + return nil, fmt.Errorf("inject engine config: pod already has a container named %q that this adapter did not render; that name is reserved for the LMCache MP native sidecar — rename your container", want.Name) + } + containers[i] = want + return containers, nil + } + return append(containers, want), nil +} + +func adoptVolume(volumes []corev1.Volume, want corev1.Volume, owned bool) ([]corev1.Volume, error) { + for i := range volumes { + if volumes[i].Name != want.Name { + continue + } + if !owned { + return nil, fmt.Errorf("inject engine config: pod already has a volume named %q that this adapter did not render; that name is reserved for the LMCache MP wire — rename your volume", want.Name) + } + volumes[i] = want + return volumes, nil + } + return append(volumes, want), nil +} + +func checkLMCacheMPShmReusable(volumes []corev1.Volume, mount corev1.VolumeMount) error { + if mount.ReadOnly { + return fmt.Errorf("inject engine config: engine container mounts %q read-only (volume %q), but the LMCache MP data path writes there — drop readOnly or mount it elsewhere", lmCacheMPShmMountPath, mount.Name) + } + if mount.SubPathExpr != "" { + return fmt.Errorf("inject engine config: engine container mounts %q with subPathExpr %q (volume %q); the LMCache MP server cannot reproduce that expansion in its own env — use a literal subPath, or mount %[1]q without it", lmCacheMPShmMountPath, mount.SubPathExpr, mount.Name) + } + for i := range volumes { + if volumes[i].Name != mount.Name { + continue + } + source := volumes[i].VolumeSource + readOnly := source.ConfigMap != nil || source.Secret != nil || source.DownwardAPI != nil || source.Projected != nil || + (source.PersistentVolumeClaim != nil && source.PersistentVolumeClaim.ReadOnly) || + (source.CSI != nil && source.CSI.ReadOnly != nil && *source.CSI.ReadOnly) || + (source.NFS != nil && source.NFS.ReadOnly) + if readOnly { + return fmt.Errorf("inject engine config: engine container mounts %q from read-only volume %q, but the LMCache MP data path writes there — use an emptyDir (medium: Memory) instead", lmCacheMPShmMountPath, mount.Name) + } + return nil + } + return nil +} + +func upsertMountByName(mounts []corev1.VolumeMount, want corev1.VolumeMount) []corev1.VolumeMount { + for i := range mounts { + if mounts[i].Name == want.Name { + mounts[i] = want + return mounts + } + } + return append(mounts, want) +} + +func splitLMCacheHostPort(s string) (host, port string, hasPort bool) { + if s == "" { + return "", "", false + } + if strings.HasPrefix(s, "[") { + end := strings.Index(s, "]") + if end <= 1 { + return "", "", false + } + host = s[1:end] + tail := s[end+1:] + if tail == "" { + return host, "", false + } + if !strings.HasPrefix(tail, ":") || strings.Contains(tail[1:], ":") { + return "", "", false + } + return host, tail[1:], true + } + if strings.Count(s, ":") > 1 { + return "", "", false + } + if i := strings.LastIndex(s, ":"); i >= 0 { + return s[:i], s[i+1:], true + } + return s, "", false +} diff --git a/internal/adapters/builtin/runtime/runtime_helpers_unit_test.go b/internal/adapters/builtin/runtime/runtime_helpers_unit_test.go new file mode 100644 index 00000000..c5ffaf3a --- /dev/null +++ b/internal/adapters/builtin/runtime/runtime_helpers_unit_test.go @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +func TestRuntimeHelpersCurrentBranches(t *testing.T) { + cache := &cachev1alpha1.CacheBackend{} + pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine"}}} + for name, err := range map[string]error{ + "nil pod": validateInjectPodCacheInputs(nil, cache, "engine"), + "nil cache": validateInjectPodCacheInputs(pod, nil, "engine"), + "no containers": validateInjectPodCacheInputs(&corev1.PodSpec{}, cache, "engine"), + } { + if err == nil { + t.Fatalf("%s: expected error", name) + } + } + + no := false + cache.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ + Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, + FailOpen: &no, + } + if got := FailOpenString(cache); got != "false" { + t.Fatalf("FailOpenString() = %q", got) + } + if got := IntegrationRole(cache); got != cachev1alpha1.CacheBackendIntegrationRoleReadWrite { + t.Fatalf("IntegrationRole() = %q", got) + } + + if got := UpsertArgPair([]string{"--port"}, "--port", "2"); len(got) != 2 || got[1] != "2" { + t.Fatalf("append missing pair value = %v", got) + } + if got := UpsertArgPair([]string{"before", "--port=1", "after"}, "--port", "2"); len(got) != 4 || got[1] != "--port" || got[2] != "2" { + t.Fatalf("replace equals pair = %v", got) + } + ref := &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}} + if got := UpsertEnv([]corev1.EnvVar{{Name: "A", Value: "old"}}, corev1.EnvVar{Name: "A", ValueFrom: ref}); got[0].Value != "" || got[0].ValueFrom == nil { + t.Fatalf("replace env = %+v", got) + } + + uid, gid := int64(1000), int64(2000) + nonRoot := true + security := lmCacheMPServerSecurityContext(&corev1.SecurityContext{RunAsUser: &uid, RunAsGroup: &gid, RunAsNonRoot: &nonRoot}) + if security.RunAsUser == nil || *security.RunAsUser != uid || security.RunAsGroup == nil || *security.RunAsGroup != gid || security.RunAsNonRoot == nil || !*security.RunAsNonRoot { + t.Fatalf("security context did not inherit engine identity: %+v", security) + } +} + +func TestCheckLMCacheMPShmReusableBranches(t *testing.T) { + readOnly := true + tests := []struct { + name string + mount corev1.VolumeMount + source corev1.VolumeSource + wantErr string + }{ + {name: "read-only mount", mount: corev1.VolumeMount{Name: "shm", ReadOnly: true}, wantErr: "read-only"}, + {name: "subPathExpr", mount: corev1.VolumeMount{Name: "shm", SubPathExpr: "$(POD_NAME)"}, wantErr: "subPathExpr"}, + {name: "config map", mount: corev1.VolumeMount{Name: "shm"}, source: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}, wantErr: "read-only volume"}, + {name: "read-only CSI", mount: corev1.VolumeMount{Name: "shm"}, source: corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{ReadOnly: &readOnly}}, wantErr: "read-only volume"}, + {name: "writable", mount: corev1.VolumeMount{Name: "shm"}, source: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {name: "missing volume", mount: corev1.VolumeMount{Name: "missing"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var volumes []corev1.Volume + if tc.mount.Name == "shm" { + volumes = []corev1.Volume{{Name: "shm", VolumeSource: tc.source}} + } + err := checkLMCacheMPShmReusable(volumes, tc.mount) + if tc.wantErr == "" && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("error = %v, want %q", err, tc.wantErr) + } + }) + } +} + +func TestSplitLMCacheHostPort(t *testing.T) { + for _, tc := range []struct { + input, host, port string + hasPort bool + }{ + {input: ""}, + {input: "redis", host: "redis"}, + {input: "redis:6379", host: "redis", port: "6379", hasPort: true}, + {input: "[2001:db8::1]", host: "2001:db8::1"}, + {input: "[2001:db8::1]:6379", host: "2001:db8::1", port: "6379", hasPort: true}, + {input: "[x"}, + {input: "[::1]bad"}, + {input: "[::1]:1:2"}, + {input: "2001:db8::1"}, + } { + host, port, hasPort := splitLMCacheHostPort(tc.input) + if host != tc.host || port != tc.port || hasPort != tc.hasPort { + t.Errorf("splitLMCacheHostPort(%q) = (%q, %q, %t), want (%q, %q, %t)", tc.input, host, port, hasPort, tc.host, tc.port, tc.hasPort) + } + } +} + +func TestLMCacheMPRendererValidationBranches(t *testing.T) { + valid := lmCacheMPServerConfig{Image: "lmcache:test", Port: 5555, ChunkSizeTokens: 256, L1Capacity: resource.MustParse("1Gi"), MaxWorkers: 1} + for _, mutate := range []func(*lmCacheMPServerConfig){ + func(c *lmCacheMPServerConfig) { c.Image = " " }, + func(c *lmCacheMPServerConfig) { c.Port = 0 }, + func(c *lmCacheMPServerConfig) { c.Port = lmCacheMPHTTPPort }, + func(c *lmCacheMPServerConfig) { c.ChunkSizeTokens = 0 }, + func(c *lmCacheMPServerConfig) { c.L1Capacity = resource.Quantity{} }, + func(c *lmCacheMPServerConfig) { c.MaxWorkers = 0 }, + } { + cfg := valid + mutate(&cfg) + if err := validateLMCacheMPServerConfig(cfg); err == nil { + t.Fatalf("expected invalid config error: %+v", cfg) + } + } + + if _, _, err := renderLMCacheMPL2Binding(nil); err != nil { + t.Fatalf("nil binding: %v", err) + } + for _, binding := range []*backendadapter.Binding{ + {Protocol: backendadapter.Protocol("other"), Endpoint: "redis:6379"}, + {Protocol: backendadapter.ProtocolRESP, Endpoint: "redis"}, + {Protocol: backendadapter.ProtocolRESP, Endpoint: "redis:not-a-port"}, + {Protocol: backendadapter.ProtocolRESP, Endpoint: "redis:70000"}, + } { + if _, _, err := renderLMCacheMPL2Binding(binding); err == nil { + t.Fatalf("expected invalid binding error: %+v", binding) + } + } + + if _, err := quantityAsGiB(resource.Quantity{}); err == nil { + t.Fatal("expected zero quantity error") + } + if got, err := quantityAsGiB(resource.MustParse("1536Mi")); err != nil || got != "1.5" { + t.Fatalf("quantityAsGiB() = %q, %v", got, err) + } + if lmCacheMPWireIsOurs(nil) { + t.Fatal("nil pod cannot contain an owned MP wire") + } +} diff --git a/internal/adapters/builtin/runtime/sglang_hicache.go b/internal/adapters/builtin/runtime/sglang_hicache.go index 9fb64d50..cf77e44f 100644 --- a/internal/adapters/builtin/runtime/sglang_hicache.go +++ b/internal/adapters/builtin/runtime/sglang_hicache.go @@ -253,9 +253,6 @@ func resolveHiCacheConfig(cache *cachev1alpha1.CacheBackend) (resolvedHiCacheCon return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: integration.failOpen must be true") } } - if cache.Spec.Autoscaling != nil { - return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: autoscaling is unsupported for an engine-local backend") - } if cache.Spec.EngineSelector == nil || len(cache.Spec.EngineSelector.MatchLabels) == 0 { return resolvedHiCacheConfig{}, fmt.Errorf("resolve SGLang HiCache config: spec.engineSelector.matchLabels is required") } diff --git a/internal/adapters/builtin/runtime/sglang_hicache_test.go b/internal/adapters/builtin/runtime/sglang_hicache_test.go index a98004af..b41ad593 100644 --- a/internal/adapters/builtin/runtime/sglang_hicache_test.go +++ b/internal/adapters/builtin/runtime/sglang_hicache_test.go @@ -254,9 +254,6 @@ func TestHiCacheRejectsInvalidBackendAtAdapterBoundary(t *testing.T) { {"fail closed", func(cache *cachev1alpha1.CacheBackend) { cache.Spec.Integration.FailOpen = &falseValue }}, - {"autoscaling", func(cache *cachev1alpha1.CacheBackend) { - cache.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 2} - }}, {"missing selector", func(cache *cachev1alpha1.CacheBackend) { cache.Spec.EngineSelector = nil }}, diff --git a/internal/adapters/builtin/runtime/sglang_lmcache.go b/internal/adapters/builtin/runtime/sglang_lmcache.go index 245f0df8..a4a8b9e6 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache.go @@ -16,6 +16,13 @@ import ( ) const ( + EnvLMCacheUseExperimental = "LMCACHE_USE_EXPERIMENTAL" + lmcacheUseExperimentalVal = "True" + SGLangEngineContainerName = "sglang" + SGLangEnableLMCacheArg = "--enable-lmcache" + SGLangEnableMetricsArg = "--enable-metrics" + SGLangConfigFileArg = "--lmcache-config-file" + // subscriberHashScheme is the canonical hash-scheme tag the SGLang // subscriber carries. Kept distinct from the runtime id and from vLLM's // "vllm" tag: the cache plane keys the index on (tenant, model, @@ -47,18 +54,17 @@ const ( // sglangLMCacheAdapter wires SGLang engine pods to LMCache for the (SGLang, LMCache) // pair. SGLang drives LMCache in MULTIPROCESS (MP) mode: // -// - Typed PodLocal objects use the shared CacheBackend-owned MP-server native +// - Typed PodLocal objects use the shared CacheBackend-configured MP-server native // sidecar + a config file (mp_host/mp_port) the engine reads via // --lmcache-config-file. A nil binding is L1-only; an optional RESP binding -// offloads to independently selected Redis storage. Topology-less legacy -// objects retain the prior SGLang-specific worker during compatibility. +// offloads to independently selected Redis storage. // - It turns LMCache on with // --enable-lmcache + LMCACHE_USE_EXPERIMENTAL (not vLLM's --kv-transfer-config) -// and does NOT inject the lm:// LMCACHE_REMOTE_URL env, which MP mode ignores. +// and does not inject any IP-connector environment. // See InjectSGLangLMCache. // -// The legacy SGLang spike was GPU-validated; the typed common-renderer path is -// intentionally not production-claimed until the Phase 3 GPU matrix passes. +// The typed common-renderer path has been GPU-validated for the supported TP=1 +// SGLang configuration. // The kvevent-subscriber sidecar rendering remains engine-agnostic. type sglangLMCacheAdapter struct { subscriber SubscriberConfig @@ -71,14 +77,18 @@ func NewSGLangLMCacheAdapter(subscriber SubscriberConfig) runtimeadapter.KVCache // Supports matches SGLang engines against an LMCache CacheBackend. Every other // (runtime, backend) combination is left for another adapter — vLLM+LMCache, -// an externally owned remote binding, or a future SGLang+Mooncake binding — and an +// an externally owned Redis binding — and an // unsupported pair surfaces as ErrNoAdapter at admission. func (sglangLMCacheAdapter) Supports(runtime runtimeadapter.RuntimeID, cache *cachev1alpha1.CacheBackend) bool { if cache == nil { return false } - return runtime == runtimeadapter.RuntimeSGLang && - cache.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache + if runtime != runtimeadapter.RuntimeSGLang || + cache.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { + return false + } + return cache.Spec.IsEventsOnly() || + (cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal) } // SupportedPairs lets the registry surface this adapter's canonical pair in the @@ -98,20 +108,7 @@ func (sglangLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) boo // InjectEngineConfig renders SGLang's LMCache MP-mode launch surface from a // host-only nil binding or a RESP binding for Redis L2 storage. func (a sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { - var err error - if cache != nil && cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" { - err = injectSGLangLMCachePodLocal(pod, binding, cache) - } else { - endpoint := "" - if binding != nil { - if binding.Protocol != backendadapter.ProtocolRESP { - return fmt.Errorf("SGLang LMCache adapter does not support remote binding protocol %q", binding.Protocol) - } - endpoint = binding.Endpoint - } - err = InjectSGLangLMCache(pod, endpoint, cache) - } - if err != nil { + if err := injectSGLangLMCachePodLocal(pod, binding, cache); err != nil { return err } return ensureSGLangMetricsForSubscriber(pod, cache, a.subscriber) @@ -159,9 +156,6 @@ func (sglangLMCacheAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a ); err != nil { return err } - if findContainerByName(pod.Spec.InitContainers, sglangMPWorkerContainerName) != nil { - return fmt.Errorf("legacy LMCache MP sidecar %q is present; recreate the Pod from an un-injected template before enabling typed PodLocal", sglangMPWorkerContainerName) - } return nil } @@ -300,7 +294,7 @@ func (sglangLMCacheAdapter) ReservedArgs() []string { // ReservedEnv returns the env var names this adapter injects and blocks // engineOverrides from touching. SGLang drives LMCache in MP mode (config-file + -// node-local worker), so — unlike the old lm:// wire — LMCACHE_REMOTE_URL and the +// node-local worker), so legacy IP-connector environment and the // serde/local-CPU tunables are NOT injected and NOT reserved. What remains: // // - LMCACHE_USE_EXPERIMENTAL (set to "True") gates SGLang's experimental LMCache diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index 33f064fd..2bf38ac2 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -8,7 +8,6 @@ import ( "flag" "io" "reflect" - "strconv" "strings" "testing" @@ -17,108 +16,89 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - provideradapter "github.com/cachebox-project/inference-cache/internal/adapters/builtin/storage" "github.com/cachebox-project/inference-cache/internal/enginebinding" backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" runtimeadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" ) -func newSGLangBackend(cfg map[string]string) *cachev1alpha1.CacheBackend { - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - LMCache: &cachev1alpha1.LMCacheEngineSpec{}, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{}, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{Image: cfg["redisImage"]}, - }, - }, - } - if value := cfg["chunkSize"]; value != "" { - parsed, _ := strconv.ParseInt(value, 10, 32) - chunkSize := int32(parsed) - cb.Spec.LMCache.ChunkSizeTokens = &chunkSize - } - cb.Spec.LMCache.WorkerImage = cfg["workerImage"] - if value := cfg["mpPort"]; value != "" { - parsed, _ := strconv.ParseInt(value, 10, 32) - port := int32(parsed) - cb.Spec.LMCache.WorkerPort = &port - } - if value := cfg["l1SizeGB"]; value != "" { - if capacity, err := resource.ParseQuantity(value + "Gi"); err == nil { - cb.Spec.LMCache.HostMemory = &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &capacity} - } - } - if value := cfg["model"]; value != "" { - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: value} - } - return cb -} - -func newTypedSGLangMPBackend() *cachev1alpha1.CacheBackend { - chunkSize := int32(256) +func typedSGLangBackend() *cachev1alpha1.CacheBackend { return &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, Type: cachev1alpha1.CacheBackendTypeLMCache, LMCache: &cachev1alpha1.LMCacheEngineSpec{ - Topology: cachev1alpha1.LMCacheTopologyPodLocal, - ChunkSizeTokens: &chunkSize, + Topology: cachev1alpha1.LMCacheTopologyPodLocal, PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ - Image: testLMCacheServerImage, + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Port: 6500, L1Capacity: resource.MustParse("4Gi"), - MaxWorkers: 2, + MaxWorkers: 4, Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, - Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, }, }}, }, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{}, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, }, } } -func respBinding(endpoint string) *backendadapter.Binding { - return &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: endpoint} +func observedTypedSGLangBackend(model string) *cachev1alpha1.CacheBackend { + backend := typedSGLangBackend() + backend.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: model} + return backend } -func findInitContainer(cs []corev1.Container, name string) *corev1.Container { - for i := range cs { - if cs[i].Name == name { - return &cs[i] - } +func TestSGLangLMCacheSelectsOnlyTypedPodLocal(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) + cache := typedSGLangBackend() + if !adapter.Supports(runtimeadapter.RuntimeSGLang, cache) { + t.Fatal("typed PodLocal SGLang backend was not selected") } - return nil -} - -func findVolume(vs []corev1.Volume, name string) *corev1.Volume { - for i := range vs { - if vs[i].Name == name { - return &vs[i] - } + cache.Spec.LMCache.Topology = "" + if adapter.Supports(runtimeadapter.RuntimeSGLang, cache) { + t.Fatal("topology-less SGLang backend selected after legacy removal") + } + cache.Spec.LMCache = nil + cache.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly + if !adapter.Supports(runtimeadapter.RuntimeSGLang, cache) { + t.Fatal("events-only SGLang backend was not selected for subscriber wiring") } - return nil } -func hasMount(ms []corev1.VolumeMount, name string) bool { - for _, m := range ms { - if m.Name == name { - return true - } +func TestSGLangLMCacheInjectsTypedMP(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) + cache := typedSGLangBackend() + pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "engine:test", Args: []string{"serve", "model", "--page-size", "64"}}}} + binding := &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: "redis.ns1.svc.cluster.local:6379"} + if err := adapter.InjectEngineConfig(pod, binding, cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + engine := pod.Containers[0] + if !hasArg(engine.Args, SGLangEnableLMCacheArg) || !hasArg(engine.Args, SGLangConfigFileArg) { + t.Fatalf("engine args = %v, want typed MP flags", engine.Args) + } + if len(pod.InitContainers) != 1 || pod.InitContainers[0].Name != lmCacheMPServerContainerName { + t.Fatalf("init containers = %+v, want MP server native sidecar", pod.InitContainers) + } + if got := strings.Join(pod.InitContainers[0].Args, " "); !strings.Contains(got, "resp") { + t.Fatalf("MP server args = %q, want Redis RESP L3 binding", got) } - return false } -func resolveRedisServer(_ runtimeadapter.KVCacheRuntimeAdapter, cb *cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { - return provideradapter.ResolveRedisL2Server(cb) +func TestSGLangLMCacheValidatesPageSize(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + cache := typedSGLangBackend() + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}}}}} + if err := adapter.ValidateMPEnginePod(pod, cache); err != nil { + t.Fatalf("ValidateMPEnginePod: %v", err) + } + pod.Spec.Containers[0].Args = []string{"--page-size", "100"} + if err := adapter.ValidateMPEnginePod(pod, cache); err == nil { + t.Fatal("ValidateMPEnginePod accepted page size that does not divide chunk size") + } } func TestSGLangSupports(t *testing.T) { @@ -129,8 +109,8 @@ func TestSGLangSupports(t *testing.T) { cache *cachev1alpha1.CacheBackend want bool }{ - {"sglang+lmcache", runtimeadapter.RuntimeSGLang, newSGLangBackend(nil), true}, - {"vllm+lmcache", runtimeadapter.RuntimeVLLM, newSGLangBackend(nil), false}, + {"sglang+lmcache", runtimeadapter.RuntimeSGLang, typedSGLangBackend(), true}, + {"vllm+lmcache", runtimeadapter.RuntimeVLLM, typedSGLangBackend(), false}, {"sglang+unsupported", runtimeadapter.RuntimeSGLang, &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{Type: cachev1alpha1.CacheBackendType("unsupported")}}, false}, {"nil cache", runtimeadapter.RuntimeSGLang, nil, false}, } @@ -154,126 +134,8 @@ func TestSGLangSupportedPairs(t *testing.T) { } } -func TestSGLangResolveCacheServer(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - pod, svc, err := resolveRedisServer(a, newSGLangBackend(nil)) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if pod == nil || svc == nil { - t.Fatalf("ResolveCacheServer returned nil pod or svc") - } - // SGLang MP mode offloads to a shared Redis L2 (not the lm:// server — lm:// - // is not a valid MP --l2-adapter type). The exhaustive render edge-cases live - // in the runtime package's redis_l2_test.go; here we pin that a (sglang, - // LMCache) backend provisions the Redis L2 store. - if len(pod.Containers) != 1 || pod.Containers[0].Name != "redis-l2" { - t.Fatalf("containers = %+v, want a single redis-l2", pod.Containers) - } - if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 6379 { - t.Fatalf("svc ports = %v, want a single 6379 port", svc.Spec.Ports) - } -} - -func TestSGLangResolveCacheServerImageOverride(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(map[string]string{"redisImage": "registry.example.com/redis:pinned"}) - pod, _, err := resolveRedisServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if got := pod.Containers[0].Image; got != "registry.example.com/redis:pinned" { - t.Fatalf("image = %q, want overridden", got) - } -} - -func TestSGLangResolveCacheServerNilCache(t *testing.T) { - if _, _, err := resolveRedisServer(NewSGLangLMCacheAdapter(SubscriberConfig{}), nil); err == nil { - t.Fatalf("ResolveCacheServer(nil) returned no error") - } -} - -func TestSGLangCanonicalHostOnlyBindingDoesNotSelectRedis(t *testing.T) { - chunkSize := int32(128) - capacity := resource.MustParse("6Gi") - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - LMCache: &cachev1alpha1.LMCacheEngineSpec{ - ChunkSizeTokens: &chunkSize, - HostMemory: &cachev1alpha1.CacheBackendHostMemorySpec{ - Capacity: &capacity, - }, - }, - }, - } - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "sglang", Image: "sglang:test"}}} - adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) - if !adapter.SupportsBinding(nil) { - t.Fatal("SGLang LMCache adapter rejected host-only binding") - } - if err := adapter.InjectEngineConfig(pod, nil, cache); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - worker := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if worker == nil { - t.Fatal("LMCache MP worker was not injected") - } - script := worker.Args[0] - if strings.Contains(script, "--l2-adapter") || strings.Contains(script, "redis") { - t.Fatalf("host-only worker command selected remote storage: %q", script) - } - if !strings.Contains(script, "--chunk-size 128") || !strings.Contains(script, "--l1-size-gb 6") { - t.Fatalf("worker command did not consume typed LMCache config: %q", script) - } -} - -func TestSGLangTypedPodLocalUsesCommonRenderer(t *testing.T) { - adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) - if _, ok := adapter.(runtimeadapter.LMCacheMPRuntimeAdapter); !ok { - t.Fatalf("adapter %T does not implement LMCacheMPRuntimeAdapter", adapter) - } - - cache := newTypedSGLangMPBackend() - pod := &corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, - }}} - if err := adapter.InjectEngineConfig(pod, respBinding("redis.ns1.svc:6379"), cache); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - server := findInitContainer(pod.InitContainers, lmCacheMPServerContainerName) - if server == nil { - t.Fatalf("typed MP server missing: %+v", pod.InitContainers) - } - if server.Image != testLMCacheServerImage || server.Image == pod.Containers[0].Image { - t.Fatalf("server image = %q, engine image = %q", server.Image, pod.Containers[0].Image) - } - joined := strings.Join(append(server.Command, server.Args...), " ") - if !strings.Contains(joined, "lmcache server") || strings.Contains(joined, "python3 -m") { - t.Fatalf("typed server entrypoint = %s", joined) - } - engine := pod.Containers[0] - if !containsArg(engine.Args, SGLangEnableLMCacheArg) || !containsArg(engine.Args, SGLangConfigFileArg) { - t.Fatalf("SGLang typed launch args missing: %v", engine.Args) - } - configIndex := -1 - for i := range engine.Args { - if engine.Args[i] == SGLangConfigFileArg { - configIndex = i - break - } - } - if configIndex < 0 || configIndex+1 >= len(engine.Args) || engine.Args[configIndex+1] != lmCacheMPConfigFilePath { - t.Fatalf("SGLang config path = %v, want %q", engine.Args, lmCacheMPConfigFilePath) - } - if findInitContainer(pod.InitContainers, sglangMPWorkerContainerName) != nil { - t.Fatalf("typed path also injected legacy worker: %+v", pod.InitContainers) - } -} - func TestSGLangInjectsMetricsOnlyWhenSubscriberWillAttach(t *testing.T) { - cache := newTypedSGLangMPBackend() + cache := typedSGLangBackend() cache.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "gemma"} pod := &corev1.PodSpec{Containers: []corev1.Container{{ Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, @@ -286,7 +148,7 @@ func TestSGLangInjectsMetricsOnlyWhenSubscriberWillAttach(t *testing.T) { t.Fatalf("engine args missing %s required by subscriber: %v", SGLangEnableMetricsArg, pod.Containers[0].Args) } - withoutSubscriber := newTypedSGLangMPBackend() + withoutSubscriber := typedSGLangBackend() pod = &corev1.PodSpec{Containers: []corev1.Container{{ Name: SGLangEngineContainerName, Image: "sglang:connector-ready", Args: []string{"--model", "gemma"}, }}} @@ -298,25 +160,6 @@ func TestSGLangInjectsMetricsOnlyWhenSubscriberWillAttach(t *testing.T) { } } -func TestSGLangValidateTypedMPEnginePod(t *testing.T) { - adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) - cache := newTypedSGLangMPBackend() - pod := &corev1.Pod{ - Spec: corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, - Args: []string{"--page-size=1"}, - }}}, - } - if err := adapter.ValidateMPEnginePod(pod, cache); err != nil { - t.Fatalf("ValidateMPEnginePod: %v", err) - } - - pod.Spec.InitContainers = []corev1.Container{{Name: sglangMPWorkerContainerName}} - if err := adapter.ValidateMPEnginePod(pod, cache); err == nil || !strings.Contains(err.Error(), "legacy") { - t.Fatalf("legacy collision error = %v", err) - } -} - func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) tests := []struct { @@ -339,7 +182,7 @@ func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { Name: SGLangEngineContainerName, Args: tc.args, }}}} - err := adapter.ValidateMPEnginePod(pod, newTypedSGLangMPBackend()) + err := adapter.ValidateMPEnginePod(pod, typedSGLangBackend()) if tc.wantErr == "" { if err != nil { t.Fatalf("ValidateMPEnginePod: %v", err) @@ -353,838 +196,9 @@ func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { } } -func TestSGLangInjectEngineConfig(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) - pod := &corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: SGLangEngineContainerName, - Image: "sglang:test", - Args: []string{"--page-size", "64"}, - Env: []corev1.EnvVar{{Name: "HF_TOKEN", Value: "secret-token"}}, - }, - { - Name: "sidecar", - Env: []corev1.EnvVar{{Name: "SIDECAR_VAR", Value: "untouched"}}, - }, - }, - } - - if err := a.InjectEngineConfig(pod, respBinding("cache.ns1.svc.cluster.local:6379"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - - engine := pod.Containers[0] - - // MP-mode engine wire: connector on + config-file, and the lm:// env is GONE. - if !containsArg(engine.Args, SGLangEnableLMCacheArg) { - t.Fatalf("engine args missing %s: %v", SGLangEnableLMCacheArg, engine.Args) - } - if !containsArg(engine.Args, SGLangEnableMetricsArg) { - t.Fatalf("engine args missing %s (SGLang metrics must be on for the scraper): %v", SGLangEnableMetricsArg, engine.Args) - } - if !containsArg(engine.Args, SGLangConfigFileArg) { - t.Fatalf("engine args missing %s: %v", SGLangConfigFileArg, engine.Args) - } - if v, ok := lookupEnv(engine.Env, EnvLMCacheUseExperimental); !ok || v != "True" { - t.Fatalf("%s = (%q, %v), want True", EnvLMCacheUseExperimental, v, ok) - } - if v, ok := lookupEnv(engine.Env, EnvInferenceCacheFailOpen); !ok || v == "" { - t.Fatalf("%s missing", EnvInferenceCacheFailOpen) - } - // The old lm:// env is NOT injected — SGLang MP mode ignores it. - if _, ok := lookupEnv(engine.Env, EnvLMCacheRemoteURL); ok { - t.Fatalf("%s injected — SGLang MP mode must not use the lm:// env", EnvLMCacheRemoteURL) - } - // vLLM-only env/args stay absent. - if _, ok := lookupEnv(engine.Env, EnvVLLMUseV1); ok { - t.Fatalf("%s (vLLM-only) injected for SGLang", EnvVLLMUseV1) - } - if _, ok := lookupEnv(engine.Env, EnvPythonHashSeed); ok { - t.Fatalf("%s (vLLM-only) injected for SGLang", EnvPythonHashSeed) - } - if containsArg(engine.Args, "--kv-transfer-config") { - t.Fatalf("--kv-transfer-config (vLLM-only) injected for SGLang: %v", engine.Args) - } - // Existing args/env preserved. - if !containsArg(engine.Args, "--page-size") { - t.Fatalf("--page-size was dropped: %v", engine.Args) - } - if v, _ := lookupEnv(engine.Env, "HF_TOKEN"); v != "secret-token" { - t.Fatalf("HF_TOKEN clobbered: got %q", v) - } - - // The MP-worker native sidecar is injected (an initContainer, restartPolicy - // Always), version-aligned to the engine image, GPU-visible but GPU-less, and - // pointing its resp --l2-adapter at the endpoint Redis. - worker := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if worker == nil { - t.Fatalf("MP-worker sidecar not injected: initContainers = %+v", pod.InitContainers) - } - if worker.RestartPolicy == nil || *worker.RestartPolicy != corev1.ContainerRestartPolicyAlways { - t.Fatalf("worker is not a native sidecar (restartPolicy Always): %v", worker.RestartPolicy) - } - if worker.Image != "sglang:test" { - t.Fatalf("worker image = %q, want the engine image (version-aligned default)", worker.Image) - } - if v, _ := lookupEnv(worker.Env, "NVIDIA_VISIBLE_DEVICES"); v != "all" { - t.Fatalf("worker NVIDIA_VISIBLE_DEVICES = %q, want all (GPU-less sidecar must see the GPU for CUDA-IPC)", v) - } - joined := strings.Join(worker.Args, " ") - for _, want := range []string{`"type":"resp"`, `"host":"cache.ns1.svc.cluster.local"`, `"port":6379`} { - if !strings.Contains(joined, want) { - t.Fatalf("worker --l2-adapter missing %s: %s", want, joined) - } - } - - // Shared volumes (/dev/shm memory-backed) + engine mounts. - if findVolume(pod.Volumes, "lmcache-config") == nil { - t.Fatalf("config volume missing: %v", pod.Volumes) - } - shm := findVolume(pod.Volumes, "lmcache-dshm") - if shm == nil || shm.EmptyDir == nil || shm.EmptyDir.Medium != corev1.StorageMediumMemory { - t.Fatalf("/dev/shm is not a memory-backed emptyDir: %+v", shm) - } - if !hasMount(engine.VolumeMounts, "lmcache-config") || !hasMount(engine.VolumeMounts, "lmcache-dshm") { - t.Fatalf("engine volume mounts missing config/dshm: %v", engine.VolumeMounts) - } - - // The non-engine sidecar container is untouched. - if len(pod.Containers[1].Env) != 1 || pod.Containers[1].Env[0].Name != "SIDECAR_VAR" { - t.Fatalf("non-engine sidecar was mutated: %v", pod.Containers[1].Env) - } -} - -func TestSGLangInjectEngineConfigSingleContainerPodAcceptsAnyName(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine", Image: "img"}}} - if err := a.InjectEngineConfig(pod, respBinding("cache.ns1.svc:6379"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - if !containsArg(pod.Containers[0].Args, SGLangConfigFileArg) { - t.Fatalf("single-container pod missing %s; should have been treated as the engine", SGLangConfigFileArg) - } -} - -func TestSGLangInjectEngineConfigMultiContainerWithoutSGLangNameErrors(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{ - {Name: "engine"}, - {Name: "sidecar"}, - }} - err := a.InjectEngineConfig(pod, respBinding("cache.ns1.svc:65432"), cb) - if err == nil { - t.Fatalf("expected an error for multi-container pod without an sglang-named container") - } - for _, c := range pod.Containers { - if _, ok := lookupEnv(c.Env, EnvLMCacheRemoteURL); ok { - t.Fatalf("container %q got env injected before the error: %v", c.Name, c.Env) - } - } -} - -func TestSGLangInjectEngineConfigIdempotent(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "img"}}} - if err := a.InjectEngineConfig(pod, respBinding("first.svc:6379"), cb); err != nil { - t.Fatalf("first InjectEngineConfig: %v", err) - } - if err := a.InjectEngineConfig(pod, respBinding("second.svc:6379"), cb); err != nil { - t.Fatalf("second InjectEngineConfig: %v", err) - } - // --enable-lmcache appears exactly once (no duplicate on re-inject). - flags := 0 - for _, arg := range pod.Containers[0].Args { - if arg == SGLangEnableLMCacheArg { - flags++ - } - } - if flags != 1 { - t.Fatalf("%s count = %d, want 1", SGLangEnableLMCacheArg, flags) - } - // Exactly one worker sidecar and two volumes (config + dshm) — re-inject - // upserts by name rather than appending duplicates. - workers := 0 - for _, c := range pod.InitContainers { - if c.Name == "lmcache-mp-worker" { - workers++ - } - } - if workers != 1 { - t.Fatalf("worker sidecar count = %d, want 1", workers) - } - if len(pod.Volumes) != 2 { - t.Fatalf("volume count = %d, want 2 (config + dshm, not duplicated): %v", len(pod.Volumes), pod.Volumes) - } - // Re-inject updates the worker's L2 endpoint in place (second wins). - worker := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if worker == nil || !strings.Contains(strings.Join(worker.Args, " "), `"host":"second.svc"`) { - t.Fatalf("re-inject did not update the worker's L2 endpoint: %+v", worker) - } -} - -func TestSGLangInjectEngineConfigConfigOverrides(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(map[string]string{ - "chunkSize": "512", - "l1SizeGB": "8", - "workerImage": "registry.example/lmcache-worker:pinned", - "mpPort": "6000", - }) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "img"}}} - if err := a.InjectEngineConfig(pod, respBinding("x.svc:6379"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - worker := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if worker == nil { - t.Fatalf("no worker sidecar") - } - if worker.Image != "registry.example/lmcache-worker:pinned" { - t.Fatalf("worker image = %q, want the workerImage override", worker.Image) - } - joined := strings.Join(worker.Args, " ") - for _, want := range []string{"--chunk-size 512", "--l1-size-gb 8", "--port 6000"} { - if !strings.Contains(joined, want) { - t.Fatalf("worker args missing %q: %s", want, joined) - } - } - if !containsArg(pod.Containers[0].Args, SGLangConfigFileArg) { - t.Fatalf("engine missing %s", SGLangConfigFileArg) - } -} - -func TestSGLangInjectEngineConfigReusesExistingDevShm(t *testing.T) { - // GPU engine manifests commonly mount their own tmpfs at /dev/shm. Appending a - // SECOND mount at the same mountPath makes the Pod invalid (the API server - // rejects duplicate mountPaths), so injection must REUSE the engine's volume for - // the worker rather than adding its own. - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - pod := &corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, - Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{{Name: "dshm", MountPath: "/dev/shm"}}, - }}, - Volumes: []corev1.Volume{{ - Name: "dshm", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}, - }}, - } - if err := a.InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - n := 0 - for _, m := range pod.Containers[0].VolumeMounts { - if m.MountPath == "/dev/shm" { - n++ - } - } - if n != 1 { - t.Fatalf("engine has %d mounts at /dev/shm, want exactly 1 (a duplicate mountPath is an invalid Pod): %+v", n, pod.Containers[0].VolumeMounts) - } - if findVolume(pod.Volumes, "lmcache-dshm") != nil { - t.Fatalf("adapter added its own lmcache-dshm volume despite the engine already mounting /dev/shm") - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - var workerShm string - for _, m := range w.VolumeMounts { - if m.MountPath == "/dev/shm" { - workerShm = m.Name - } - } - if workerShm != "dshm" { - t.Fatalf("worker /dev/shm volume = %q, want the engine's existing %q — the MP data path needs both containers on the SAME volume", workerShm, "dshm") - } -} - -func TestSGLangInjectEngineConfigRejectsConfigPathCollision(t *testing.T) { - // The config mount path is adapter-owned: the worker WRITES the MP config there. - // A pre-existing mount can neither be duplicated (invalid Pod) nor safely reused - // (a ConfigMap mount is read-only), so injection must reject with a clear reason - // — the webhook turns that into a fail-open admit. - pod := &corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, - Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{{Name: "operator-cfg", MountPath: "/etc/lmcache"}}, - }}} - err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)) - if err == nil { - t.Fatalf("want an error when the engine already mounts the adapter-owned config path") - } - if !strings.Contains(err.Error(), "/etc/lmcache") || !strings.Contains(err.Error(), "operator-cfg") { - t.Fatalf("error must name the path and the conflicting volume, got: %v", err) - } -} - -func TestSGLangInjectEngineConfigRejectsForeignReservedNames(t *testing.T) { - // Mutating admission must never erase an operator's container or volume. A - // pre-existing object carrying one of the adapter's reserved names — but NOT - // rendered by the adapter (no marker env) — is a foreign collision: reject, so - // the pod webhook fails open and the pod admits un-wired rather than corrupted. - // Silently skipping is not an option for the worker: the engine gets - // --lmcache-config-file regardless and would block on a config nothing writes. - engine := corev1.Container{Name: SGLangEngineContainerName, Image: "sglang:test"} - cases := []struct { - name string - pod *corev1.PodSpec - want string // fragment the error must name so the operator can find it - }{ - { - name: "container squats the worker name", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine}, - InitContainers: []corev1.Container{{Name: "lmcache-mp-worker", Image: "operator/own:v1"}}, - }, - want: "lmcache-mp-worker", - }, - { - name: "volume squats the config-volume name", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine}, - Volumes: []corev1.Volume{{ - Name: "lmcache-config", - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}, - }}, - }, - want: "lmcache-config", - }, - { - name: "volume squats the dshm-volume name", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine}, - Volumes: []corev1.Volume{{ - Name: "lmcache-dshm", - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/mnt/data"}}, - }}, - }, - want: "lmcache-dshm", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - before := tc.pod.DeepCopy() - err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(tc.pod, respBinding("r.svc:6379"), newSGLangBackend(nil)) - if err == nil { - t.Fatalf("want an error when %s", tc.name) - } - if !strings.Contains(err.Error(), tc.want) { - t.Fatalf("error must name the conflicting object %q, got: %v", tc.want, err) - } - // The operator's object must survive verbatim — an error that still - // clobbered the pod would defeat the purpose of rejecting. - if !reflect.DeepEqual(tc.pod.InitContainers, before.InitContainers) { - t.Fatalf("initContainers mutated despite the rejection:\n got %+v\nwant %+v", tc.pod.InitContainers, before.InitContainers) - } - if !reflect.DeepEqual(tc.pod.Volumes, before.Volumes) { - t.Fatalf("volumes mutated despite the rejection:\n got %+v\nwant %+v", tc.pod.Volumes, before.Volumes) - } - }) - } -} - -func TestSGLangInjectEngineConfigReinjectionConvergesOnCurrentRender(t *testing.T) { - // The marker env on the worker is what tells OUR container from an operator's - // squat — and it must keep doing so when the render legitimately CHANGES (a moved - // status.endpoint here). Value-equality against a fresh render would misread this - // as foreign; the second injection must instead converge the worker on the new - // endpoint rather than reject it, duplicate it, or leave the stale one. - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "img"}}} - if err := a.InjectEngineConfig(pod, respBinding("first.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("first InjectEngineConfig: %v", err) - } - if err := a.InjectEngineConfig(pod, respBinding("second.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("second InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - script := strings.Join(w.Args, " ") - if !strings.Contains(script, "second.svc") { - t.Fatalf("worker did not converge on the current endpoint; --l2-adapter still reads: %s", script) - } - if strings.Contains(script, "first.svc") { - t.Fatalf("worker kept the stale endpoint from the first injection: %s", script) - } -} - -func TestSGLangInjectEngineConfigRejectsUnwritableDevShm(t *testing.T) { - // A reused /dev/shm carries the MP data path, which WRITES. A read-only mount, or - // one backed by a projection source the kubelet always mounts read-only, would - // fail deep inside LMCache at runtime — reject at admission instead. - engine := func(m corev1.VolumeMount) corev1.Container { - return corev1.Container{ - Name: SGLangEngineContainerName, Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{m}, - } - } - cases := []struct { - name string - pod *corev1.PodSpec - want string - }{ - { - name: "read-only mount", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "dshm", MountPath: "/dev/shm", ReadOnly: true})}, - Volumes: []corev1.Volume{{ - Name: "dshm", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}, - }}, - }, - want: "read-only", - }, - { - name: "configMap-backed volume", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "shm-cfg", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "shm-cfg", - VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}, - }}, - }, - want: "configMap", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(tc.pod, respBinding("r.svc:6379"), newSGLangBackend(nil)) - if err == nil { - t.Fatalf("want an error when the engine's /dev/shm is not writable scratch (%s)", tc.name) - } - if !strings.Contains(err.Error(), "/dev/shm") || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("error must name /dev/shm and %q, got: %v", tc.want, err) - } - }) - } -} - -func TestSGLangInjectEngineConfigReusesWritableNonEmptyDirDevShm(t *testing.T) { - // The writability guard must not over-reject: a source that HAS a readOnly flag - // but leaves it false is writable, and the engine's /dev/shm must still be reused - // (a second mount at the same path is an invalid Pod). Pins that the guard keys - // on the flag's value, not on the source being exotic. - pod := &corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{{Name: "nfs-shm", MountPath: "/dev/shm"}}, - }}, - Volumes: []corev1.Volume{{ - Name: "nfs-shm", - VolumeSource: corev1.VolumeSource{NFS: &corev1.NFSVolumeSource{Server: "s", Path: "/p", ReadOnly: false}}, - }}, - } - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig rejected a writable /dev/shm: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - for _, m := range w.VolumeMounts { - if m.MountPath == "/dev/shm" && m.Name != "nfs-shm" { - t.Fatalf("worker /dev/shm volume = %q, want the engine's existing %q", m.Name, "nfs-shm") - } - } -} - -func TestSGLangInjectEngineConfigWorkerSeesTheGPU(t *testing.T) { - // GPU visibility on the worker is LOAD-BEARING, not incidental: the engine hands - // it a device UUID and LMCache's CUDA-IPC wrapper resolves that to a local index, - // which fails unless the device is visible here. GPU-validated — with visibility - // revoked the worker dies on "Device UUID not found in the discovered - // devices" and the engine never reaches ready. - // - // It cannot be narrowed to the engine's own device: the device plugin assigns the - // UUID at kubelet time, after this mutation runs. The isolation trade-off is - // documented for operators in docs/design/cachebackend-api.md. This test exists so - // the env is not dropped as dead weight — the failure it prevents is a wedged - // engine, not a cache miss. - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "sglang:test"}}} - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - if v, _ := lookupEnv(w.Env, "NVIDIA_VISIBLE_DEVICES"); v != "all" { - t.Fatalf("worker NVIDIA_VISIBLE_DEVICES = %q, want \"all\" — without it the CUDA-IPC UUID lookup fails and the engine hangs behind the startup probe", v) - } - // It must stay GPU-less at the scheduler: a device-plugin request would burn a - // second GPU and hand the worker a DIFFERENT device than the engine's. - if _, ok := w.Resources.Limits["nvidia.com/gpu"]; ok { - t.Fatalf("worker requests nvidia.com/gpu — it must consume no device-plugin allocation: %v", w.Resources.Limits) - } -} - -func TestSGLangInjectEngineConfigWorkerRestrictedSecurityContext(t *testing.T) { - // This mutation lands BEFORE Pod Security admission, so the worker must carry the - // container-only Restricted requirements itself — else it turns an admissible - // engine pod into a REJECTED one in a restricted namespace (the inverse of - // fail-open). And it must add NO capabilities (an added cap is itself a Restricted - // violation; IPC_LOCK is not needed — GPU access is via device files, not caps). - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "sglang:test"}}} - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - sc := w.SecurityContext - if sc == nil { - t.Fatalf("worker has no securityContext — a restricted namespace would reject the pod") - } - if sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation { - t.Errorf("allowPrivilegeEscalation = %v, want false (container-only Restricted requirement)", sc.AllowPrivilegeEscalation) - } - if sc.Capabilities == nil || len(sc.Capabilities.Add) > 0 { - t.Errorf("capabilities.add = %v, want none (an added cap is a Restricted violation)", sc.Capabilities) - } - dropsAll := false - if sc.Capabilities != nil { - for _, c := range sc.Capabilities.Drop { - if c == "ALL" { - dropsAll = true - } - } - } - if !dropsAll { - t.Errorf("capabilities.drop = %v, want [ALL] (container-only Restricted requirement)", sc.Capabilities) - } - if sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { - t.Errorf("seccompProfile = %v, want RuntimeDefault", sc.SeccompProfile) - } -} - -func TestSGLangInjectEngineConfigWorkerMirrorsEngineUserIdentity(t *testing.T) { - // The worker runs the operator's engine image (by default), so it must not force - // its own UID like the distroless subscriber does — it mirrors the engine's user - // identity instead, staying exactly as (non-)root as the pod was admitted to be. - nonRoot := true - uid := int64(1000) - gid := int64(2000) - pod := &corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, - Image: "sglang:test", - SecurityContext: &corev1.SecurityContext{ - RunAsNonRoot: &nonRoot, RunAsUser: &uid, RunAsGroup: &gid, - }, - }}} - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - sc := w.SecurityContext - if sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot { - t.Errorf("runAsNonRoot not mirrored from engine: %v", sc.RunAsNonRoot) - } - if sc.RunAsUser == nil || *sc.RunAsUser != uid { - t.Errorf("runAsUser = %v, want mirrored %d", sc.RunAsUser, uid) - } - if sc.RunAsGroup == nil || *sc.RunAsGroup != gid { - t.Errorf("runAsGroup = %v, want mirrored %d", sc.RunAsGroup, gid) - } - // And it does NOT force a read-only rootfs or a fixed UID when the engine sets - // none — that would risk breaking the vendor image's writes / CUDA-IPC. - pod2 := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "sglang:test"}}} - _ = NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod2, respBinding("r.svc:6379"), newSGLangBackend(nil)) - w2 := findInitContainer(pod2.InitContainers, "lmcache-mp-worker") - if w2.SecurityContext.RunAsUser != nil { - t.Errorf("runAsUser forced to %v when engine set none — must inherit from the pod, not override the image", w2.SecurityContext.RunAsUser) - } - if w2.SecurityContext.ReadOnlyRootFilesystem != nil { - t.Errorf("readOnlyRootFilesystem set — the worker writes to its rootfs; must not force it") - } -} - -func TestSGLangInjectEngineConfigMirrorsDevShmSubPath(t *testing.T) { - // Sharing the same VOLUME is not enough: if the engine mounts /dev/shm with a - // subPath and the worker mounts the volume root, the two land on DIFFERENT - // directories — the pod admits cleanly and then transfers no KV. Mirror the - // subPath so both resolve to the same place. - pod := &corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{{Name: "scratch", MountPath: "/dev/shm", SubPath: "shm"}}, - }}, - Volumes: []corev1.Volume{{ - Name: "scratch", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}, - }}, - } - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), newSGLangBackend(nil)); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - var got *corev1.VolumeMount - for i := range w.VolumeMounts { - if w.VolumeMounts[i].MountPath == "/dev/shm" { - got = &w.VolumeMounts[i] - } - } - if got == nil { - t.Fatalf("worker has no /dev/shm mount: %+v", w.VolumeMounts) - } - if got.Name != "scratch" || got.SubPath != "shm" { - t.Fatalf("worker /dev/shm = (volume %q, subPath %q), want (scratch, shm) — both containers must resolve to the SAME directory", got.Name, got.SubPath) - } -} - -func TestSGLangInjectEngineConfigRejectsUnshareableDevShm(t *testing.T) { - // Shapes the worker cannot safely share: an expansion it cannot reproduce in its - // own env, and source-level read-only that the mount-level readOnly check misses. - engine := func(m corev1.VolumeMount) corev1.Container { - return corev1.Container{ - Name: SGLangEngineContainerName, Image: "sglang:test", - VolumeMounts: []corev1.VolumeMount{m}, - } - } - csiReadOnly := true - cases := []struct { - name string - pod *corev1.PodSpec - want string - }{ - { - name: "subPathExpr cannot be reproduced in the worker's env", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "scratch", MountPath: "/dev/shm", SubPathExpr: "$(POD_NAME)/shm"})}, - Volumes: []corev1.Volume{{ - Name: "scratch", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}, - }}, - }, - want: "subPathExpr", - }, - { - name: "read-only persistentVolumeClaim source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "pvc-shm", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "pvc-shm", - VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "c", ReadOnly: true, - }}, - }}, - }, - want: "persistentVolumeClaim", - }, - { - name: "read-only csi source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "csi-shm", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "csi-shm", - VolumeSource: corev1.VolumeSource{CSI: &corev1.CSIVolumeSource{ - Driver: "d.example.com", ReadOnly: &csiReadOnly, - }}, - }}, - }, - want: "csi", - }, - // A source-level readOnly is NOT overridden by a mount-level readOnly:false, - // so every in-tree source carrying its own flag must be caught. These shapes - // are exotic at /dev/shm, but the failure they produce is the silent one. - { - name: "read-only nfs source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "nfs-shm", MountPath: "/dev/shm", ReadOnly: false})}, - Volumes: []corev1.Volume{{ - Name: "nfs-shm", - VolumeSource: corev1.VolumeSource{NFS: &corev1.NFSVolumeSource{Server: "s", Path: "/p", ReadOnly: true}}, - }}, - }, - want: "nfs", - }, - { - name: "read-only rbd source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "rbd-shm", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "rbd-shm", - VolumeSource: corev1.VolumeSource{RBD: &corev1.RBDVolumeSource{RBDImage: "i", ReadOnly: true}}, - }}, - }, - want: "rbd", - }, - { - name: "read-only cephfs source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "ceph-shm", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "ceph-shm", - VolumeSource: corev1.VolumeSource{CephFS: &corev1.CephFSVolumeSource{Monitors: []string{"m"}, ReadOnly: true}}, - }}, - }, - want: "cephfs", - }, - { - name: "read-only azureFile source", - pod: &corev1.PodSpec{ - Containers: []corev1.Container{engine(corev1.VolumeMount{Name: "az-shm", MountPath: "/dev/shm"})}, - Volumes: []corev1.Volume{{ - Name: "az-shm", - VolumeSource: corev1.VolumeSource{AzureFile: &corev1.AzureFileVolumeSource{SecretName: "s", ShareName: "sh", ReadOnly: true}}, - }}, - }, - want: "azureFile", - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(tc.pod, respBinding("r.svc:6379"), newSGLangBackend(nil)) - if err == nil { - t.Fatalf("want an error when the engine's /dev/shm is unshareable (%s)", tc.name) - } - if !strings.Contains(err.Error(), tc.want) { - t.Fatalf("error must name %q, got: %v", tc.want, err) - } - }) - } -} - -func TestSGLangInjectEngineConfigWorkerHasMemoryBudget(t *testing.T) { - // The worker holds the L1 in a memory-backed tmpfs charged to its cgroup, so it - // must carry a matching memory request+limit (l1SizeGB + 1Gi) — otherwise the L1 - // is invisible to the scheduler and can overcommit the node. - pod := &corev1.PodSpec{Containers: []corev1.Container{{ - Name: SGLangEngineContainerName, Image: "sglang:test", - }}} - cb := newSGLangBackend(map[string]string{"l1SizeGB": "8"}) - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - w := findInitContainer(pod.InitContainers, "lmcache-mp-worker") - if w == nil { - t.Fatalf("worker not injected") - } - want := resource.MustParse("9Gi") // 8Gi L1 + 1Gi headroom - req, hasReq := w.Resources.Requests[corev1.ResourceMemory] - lim, hasLim := w.Resources.Limits[corev1.ResourceMemory] - if !hasReq || !hasLim { - t.Fatalf("worker must carry a memory request AND limit, got %+v", w.Resources) - } - if req.Cmp(want) != 0 || lim.Cmp(want) != 0 { - t.Fatalf("worker memory = req %s / lim %s, want %s (l1SizeGB + 1Gi headroom)", req.String(), lim.String(), want.String()) - } - // The tmpfs sizeLimit must agree with the container budget. - v := findVolume(pod.Volumes, "lmcache-dshm") - if v == nil || v.EmptyDir == nil || v.EmptyDir.SizeLimit == nil { - t.Fatalf("/dev/shm tmpfs must be size-bounded, got %+v", v) - } - if v.EmptyDir.SizeLimit.Cmp(want) != 0 { - t.Fatalf("tmpfs sizeLimit = %s, want %s (must match the worker's memory budget)", v.EmptyDir.SizeLimit.String(), want.String()) - } -} - -func TestSGLangInjectEngineConfigSanitizesNumericConfig(t *testing.T) { - // chunkSize/mpPort/l1SizeGB flow into the worker's `sh -c` command; a - // non-positive-integer (typo, or a shell-injection attempt) MUST fall back to - // the safe default and never reach the shell verbatim. - // wantArg = the default arg that must appear; danger = the fragment that must - // NOT (asserting the bad value was not substituted, precisely — a short value - // like "0" is a substring of 127.0.0.1, so check the arg-in-context instead). - cases := []struct{ key, bad, wantArg, danger string }{ - {"chunkSize", "256; rm -rf /", "--chunk-size 256", "rm -rf"}, - {"chunkSize", "$(evil)", "--chunk-size 256", "$(evil)"}, - {"mpPort", "5555 && curl evil", "--port 5555", "curl evil"}, - {"mpPort", "-1", "--port 5555", "--port -1"}, - {"mpPort", "99999", "--port 5555", "--port 99999"}, // > 65535 → default - {"l1SizeGB", "4; cat /etc/passwd", "--l1-size-gb 4", "/etc/passwd"}, - {"l1SizeGB", "abc", "--l1-size-gb 4", "--l1-size-gb abc"}, - {"l1SizeGB", "0", "--l1-size-gb 4", "--l1-size-gb 0"}, - {"l1SizeGB", "999999999", "--l1-size-gb 4", "--l1-size-gb 999999999"}, // huge → default (bounded /dev/shm) - } - for _, tc := range cases { - t.Run(tc.key+"="+tc.bad, func(t *testing.T) { - cb := newSGLangBackend(map[string]string{tc.key: tc.bad}) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Image: "img"}}} - if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(pod, respBinding("r.svc:6379"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - joined := strings.Join(findInitContainer(pod.InitContainers, "lmcache-mp-worker").Args, " ") - if !strings.Contains(joined, tc.wantArg) { - t.Fatalf("want %q (sanitized to default); worker command: %s", tc.wantArg, joined) - } - if strings.Contains(joined, tc.danger) { - t.Fatalf("unsanitized value reached the worker shell command (%q): %s", tc.danger, joined) - } - }) - } -} - -func TestSGLangInjectEngineConfigFailOpen(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - trueVal, falseVal := true, false - cases := []struct { - name string - failOpen *bool - want string - }{ - {"default (unset → true)", nil, "true"}, - {"explicit true", &trueVal, "true"}, - {"explicit false", &falseVal, "false"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cb := newSGLangBackend(nil) - cb.Spec.Integration.FailOpen = tc.failOpen - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName}}} - if err := a.InjectEngineConfig(pod, respBinding("x.svc:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - if v, _ := lookupEnv(pod.Containers[0].Env, EnvInferenceCacheFailOpen); v != tc.want { - t.Fatalf("%s = %q, want %q", EnvInferenceCacheFailOpen, v, tc.want) - } - }) - } -} - -func TestSGLangInjectEngineConfigBadInput(t *testing.T) { - a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) - good := &corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName}}} - cases := []struct { - name string - fn func() error - }{ - {"nil pod", func() error { return a.InjectEngineConfig(nil, respBinding("x.svc:65432"), cb) }}, - {"nil cache", func() error { return a.InjectEngineConfig(good, respBinding("x.svc:65432"), nil) }}, - {"empty endpoint", func() error { return a.InjectEngineConfig(good, respBinding(""), cb) }}, - {"no containers", func() error { return a.InjectEngineConfig(&corev1.PodSpec{}, respBinding("x.svc:65432"), cb) }}, - // The resp --l2-adapter takes an INTEGER port, emitted unquoted into JSON. A - // non-numeric or out-of-range port would render invalid JSON, the worker would - // fail to parse it and never bind its ZMQ port, and the engine would sit behind - // the startup probe forever — reject at admission and let the webhook fail open. - {"non-numeric port", func() error { return a.InjectEngineConfig(good, respBinding("r.svc:redis"), cb) }}, - {"port out of range", func() error { return a.InjectEngineConfig(good, respBinding("r.svc:70000"), cb) }}, - {"zero port", func() error { return a.InjectEngineConfig(good, respBinding("r.svc:0"), cb) }}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if err := tc.fn(); err == nil { - t.Fatalf("expected error for %s, got nil", tc.name) - } - }) - } -} - func TestSGLangInjectRouterConfigIsNoop(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{}) - cb := newSGLangBackend(nil) + cb := typedSGLangBackend() pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "router", Env: []corev1.EnvVar{{Name: "EXISTING", Value: "x"}}}}} if err := a.InjectRouterConfig(pod, respBinding("x.svc:65432"), cb); err != nil { t.Fatalf("InjectRouterConfig: %v", err) @@ -1201,7 +215,7 @@ func TestSGLangInjectRouterConfigIsNoop(t *testing.T) { func TestSGLangObservationSidecarShape(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newSGLangBackend(map[string]string{"model": "Qwen/Qwen2.5-0.5B-Instruct"}) + cb := observedTypedSGLangBackend("Qwen/Qwen2.5-0.5B-Instruct") pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-a", Namespace: "engines"}} c, err := a.ObservationSidecar(cb, pod) @@ -1226,6 +240,7 @@ func TestSGLangObservationSidecarShape(t *testing.T) { // Load-bearing: the index keys on hash_scheme, so the SGLang subscriber // MUST tag its reports "sglang" to stay disjoint from vLLM entries. "--hash-scheme=sglang", + "--engine-metrics-url=http://127.0.0.1:30000/metrics", // LMCache is an L2 tier behind SGLang, same as vLLM+LMCache — drop // BlockRemoved rather than forward it as PREFIX_EVICTED. "--ignore-block-removed=true", @@ -1247,7 +262,7 @@ func TestSGLangObservationSidecarArgsParseAgainstSubscriberFlagSet(t *testing.T) // event-path flag surface and assert they parse cleanly. Keep in sync with // cmd/kvevent-subscriber/main.go. a := NewSGLangLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newSGLangBackend(map[string]string{"model": "Qwen/Qwen2.5-0.5B-Instruct"}) + cb := observedTypedSGLangBackend("Qwen/Qwen2.5-0.5B-Instruct") pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-a", Namespace: "engines"}} c, err := a.ObservationSidecar(cb, pod) if err != nil || c == nil { @@ -1282,7 +297,7 @@ func TestSGLangObservationSidecarHonoursOptions(t *testing.T) { Image: "registry.example.com/subscriber:pinned", PolicyServerGRPCAddress: "ic-server.custom-ns.svc.cluster.local:9090", }) - cb := newSGLangBackend(map[string]string{"model": "MyOrg/MyModel"}) + cb := observedTypedSGLangBackend("MyOrg/MyModel") pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-z", Namespace: "engines"}} c, err := a.ObservationSidecar(cb, pod) if err != nil || c == nil { @@ -1298,7 +313,7 @@ func TestSGLangObservationSidecarHonoursOptions(t *testing.T) { func TestSGLangObservationSidecarSkipsWithoutModel(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newSGLangBackend(nil) + cb := typedSGLangBackend() pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-a"}} c, err := a.ObservationSidecar(cb, pod) if err != nil { @@ -1311,7 +326,7 @@ func TestSGLangObservationSidecarSkipsWithoutModel(t *testing.T) { func TestSGLangObservationSidecarSkipsWithoutImage(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{}) // no image configured → auto-attach opt-out - cb := newSGLangBackend(map[string]string{"model": "MyOrg/MyModel"}) + cb := observedTypedSGLangBackend("MyOrg/MyModel") pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-a"}} c, err := a.ObservationSidecar(cb, pod) if err != nil { @@ -1324,7 +339,7 @@ func TestSGLangObservationSidecarSkipsWithoutImage(t *testing.T) { func TestSGLangObservationSidecarBadInput(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newSGLangBackend(map[string]string{"model": "m"}) + cb := observedTypedSGLangBackend("m") cases := []struct { name string cb *cachev1alpha1.CacheBackend @@ -1355,69 +370,16 @@ func TestSGLangReservedArgs(t *testing.T) { } } -func TestSGLangReservedEnv(t *testing.T) { - got := NewSGLangLMCacheAdapter(SubscriberConfig{}).ReservedEnv() - want := []string{ - EnvLMCacheUseExperimental, - EnvInferenceCacheFailOpen, - } - if len(got) != len(want) { - t.Fatalf("ReservedEnv = %v, want %v", got, want) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("ReservedEnv = %v, want %v", got, want) - } - } - // Negative control: vLLM-only env MUST NOT be reserved for SGLang (never - // injected), the LMCACHE_* tunables stay overridable, and LMCACHE_REMOTE_URL - // (the old lm:// wire) is gone in MP mode so it must not be reserved either. - forbidden := map[string]bool{ - EnvVLLMUseV1: true, - EnvPythonHashSeed: true, - EnvLMCacheChunkSize: true, - EnvLMCacheRemoteSerde: true, - EnvLMCacheRemoteURL: true, - } - for _, name := range got { - if forbidden[name] { - t.Errorf("env %q must NOT be reserved for SGLang", name) - } - } -} - func TestSGLangEngineContainerName(t *testing.T) { if got := NewSGLangLMCacheAdapter(SubscriberConfig{}).EngineContainerName(); got != SGLangEngineContainerName { t.Fatalf("EngineContainerName = %q, want %q", got, SGLangEngineContainerName) } } -// --- local test helpers (the runtime package's helpers live in a different -// test package and aren't importable here) --- - -func lookupEnv(env []corev1.EnvVar, name string) (string, bool) { - for _, e := range env { - if e.Name == name { - return e.Value, true - } - } - return "", false -} - -func containsArg(args []string, want string) bool { - for _, a := range args { - if a == want { - return true - } - } - return false -} - -func envHasFieldRef(env []corev1.EnvVar, name, path string) bool { - for _, e := range env { - if e.Name == name && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == path { - return true - } +func TestSGLangReservedEnv(t *testing.T) { + got := NewSGLangLMCacheAdapter(SubscriberConfig{}).ReservedEnv() + want := []string{EnvLMCacheUseExperimental, EnvInferenceCacheFailOpen} + if !reflect.DeepEqual(got, want) { + t.Fatalf("ReservedEnv = %v, want %v", got, want) } - return false } diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_wire.go b/internal/adapters/builtin/runtime/sglang_lmcache_wire.go deleted file mode 100644 index 9ac9237a..00000000 --- a/internal/adapters/builtin/runtime/sglang_lmcache_wire.go +++ /dev/null @@ -1,680 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package runtime - -import ( - "fmt" - "strconv" - "strings" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// SGLang engine-side wire — LMCache multiprocess (MP) mode. -// -// SGLang does NOT consume a standalone lm:// server the way vLLM does; it drives -// LMCache through a node-local MP worker. The engine attaches to that worker over -// ZMQ (mp_host/mp_port) + a shared-memory data path, configured by a -// --lmcache-config-file (the lm://-style LMCACHE_* env is ignored). The worker -// holds L1 (host memory, in /dev/shm) and offloads to a shared L2 (its -// --l2-adapter — the managed Redis this backend provisions). GPU-validated; full -// design + evidence in docs/design/sglang-lmcache-mp-mode.md. -const ( - // EnvLMCacheUseExperimental gates SGLang's experimental LMCache path; it MUST - // be "True" for --enable-lmcache to engage the connector. - EnvLMCacheUseExperimental = "LMCACHE_USE_EXPERIMENTAL" - lmcacheUseExperimentalVal = "True" - // SGLangEngineContainerName is the conventional name of the SGLang engine - // container in a pod the adapter mutates. A single-container pod is also - // treated as the engine. - SGLangEngineContainerName = "sglang" - // SGLangEnableLMCacheArg turns the LMCache connector on (a store_true flag, no - // value). Exported so the adapter can reserve it against engineOverrides. - SGLangEnableLMCacheArg = "--enable-lmcache" - // SGLangConfigFileArg points the engine at the MP config file the worker - // writes. Exported so the adapter can reserve it (suppressing it un-wires MP - // mode). - SGLangConfigFileArg = "--lmcache-config-file" - - // SGLangEnableMetricsArg turns on SGLang's Prometheus /metrics endpoint (a - // store_true flag). SGLang defaults metrics OFF, unlike vLLM, so without this - // the stats scraper has nothing to read and load-aware routing goes dark. - // The managed LMCache and HiCache paths inject it (both mutate the engine - // container) so the operator need not remember the flag. EventsOnly SGLang - // gets no engine mutation, so an operator running that mode must set it (its - // absence then surfaces as a loud stale signal, not silent zeros). - SGLangEnableMetricsArg = "--enable-metrics" - - // sglangMPWorkerContainerName is the node-local MP worker native sidecar. - sglangMPWorkerContainerName = "lmcache-mp-worker" - // envSGLangMPWorkerManaged marks the MP worker as adapter-rendered, so a - // re-injection can tell OUR container from an operator's that happens to carry - // the same name (see sglangWireIsOurs). It is inert to LMCache — an unknown env - // var the worker never reads. - envSGLangMPWorkerManaged = "INFERENCECACHE_MP_WORKER" - sglangMPWorkerManagedVal = "true" - // sglangConfigVolumeName / MountPath / FileName: the shared dir the worker - // writes the MP config into and the engine reads via --lmcache-config-file. - sglangConfigVolumeName = "lmcache-config" - sglangConfigMountPath = "/etc/lmcache" - sglangConfigFileName = "config.yaml" - // sglangShmVolumeName / MountPath: the tmpfs the MP L1 lives in. Too small - // (default 64Mi) silently falls back to slow pickle serialization, so it is - // sized from the L1 budget. - sglangShmVolumeName = "lmcache-dshm" - sglangShmMountPath = "/dev/shm" - - sglangDefaultMPPort = "5555" - sglangDefaultL1SizeGB = "4" - - // Upper bounds for the sanitized numeric tunables (see sglangIntInRangeOr). - sglangMaxChunkSize = 65536 // generous; chunk sizes are small - sglangMaxTCPPort = 65535 // a valid TCP port - sglangMaxL1SizeGB = 1024 // 1 TiB — bounded so ParseQuantity always sizes /dev/shm - - // Typed LMCache configuration keys used by the renderer. - cfgKeyChunkSize = "chunkSize" - cfgKeyWorkerImage = "workerImage" - cfgKeyL1SizeGB = "l1SizeGB" - cfgKeyMPPort = "mpPort" -) - -// InjectSGLangLMCache wires an SGLang engine pod for LMCache MP mode. It mutates -// pod in place — atomically (on error pod is untouched) and idempotently (a -// re-injection converges on the current render instead of duplicating it) — and -// adds: -// -// - a node-local MP-worker native sidecar (a restartPolicy: Always init -// container) that writes the MP config file then runs the LMCache MP server on -// 127.0.0.1, offloading to the shared L2 (resp -> the managed Redis endpoint). -// NVIDIA_VISIBLE_DEVICES=all lets the GPU-less sidecar CUDA-IPC the engine's -// GPU without consuming a device-plugin allocation; -// - shared emptyDir volumes for the config file and /dev/shm (the L1 tier); -// - on the engine container: --enable-lmcache, --lmcache-config-file, the -// LMCACHE_USE_EXPERIMENTAL + INFERENCECACHE_FAIL_OPEN env, and the shared -// volume mounts. -// -// endpoint is the managed Redis L2 address (host:port) the reconciler published to -// status.endpoint; it is used only to build the worker's resp --l2-adapter (the -// engine itself dials the local worker, never this endpoint). The engine container is -// [SGLangEngineContainerName]; a single-container pod is accepted, a -// multi-container pod with no `sglang` container is rejected. A pre-existing -// container or volume that squats one of the reserved names this wire renders — and -// that this adapter did not render (see [sglangWireIsOurs]) — is also rejected -// rather than overwritten; the pod webhook turns that into a fail-open admit. -// -// Note: unlike the old lm:// wire, this does NOT inject LMCACHE_REMOTE_URL / serde -// / local-CPU env — SGLang MP mode ignores it. -func InjectSGLangLMCache(pod *corev1.PodSpec, endpoint string, cache *cachev1alpha1.CacheBackend) error { - if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { - return err - } - if endpoint == "" && cache.Spec.EffectiveRemoteStorage() != nil { - return fmt.Errorf("inject engine config: endpoint is empty") - } - i, err := EngineContainerIndexNamed(pod, SGLangEngineContainerName) - if err != nil { - return err - } - cfg := effectiveSGLangLMCacheConfig(cache) - // SECURITY: chunkSize/mpPort/l1SizeGB are substituted into the worker's `sh -c` - // command and into resource sizing, so they MUST be plain positive integers — a - // non-integer (typo or a shell-metacharacter injection attempt) falls back to - // the safe default and never reaches the shell. sglangIntInRangeOr is the - // sanitization boundary; it also guarantees the /dev/shm sizeLimit is bounded. - chunkSize := sglangIntInRangeOr(cfg, cfgKeyChunkSize, defaultChunkSize, sglangMaxChunkSize) - mpPort := sglangIntInRangeOr(cfg, cfgKeyMPPort, sglangDefaultMPPort, sglangMaxTCPPort) - l1SizeGB := sglangIntInRangeOr(cfg, cfgKeyL1SizeGB, sglangDefaultL1SizeGB, sglangMaxL1SizeGB) - - l2Adapter := "" - if endpoint != "" { - l2Adapter, err = sglangL2AdapterJSON(endpoint) - if err != nil { - return err - } - } - - // Mutate a COPY and commit it only on success, so injection is all-or-nothing. - // Several guards below reject mid-render (a foreign name squat, an unwritable - // /dev/shm), and an in-place mutator that had already appended half the wire - // would hand the caller a pod that is neither wired nor pristine. The pod webhook - // happens to fail open with its own pre-injection copy today, but this function's - // contract should not depend on that. - work := pod.DeepCopy() - c := &work.Containers[i] - - // Did WE wire this pod already? The webhook can be handed a pod template this - // adapter has mutated before (re-admission, or an operator who copied a rendered - // spec), and re-injection must converge on the current render rather than - // duplicate or reject. Ownership is decided ONCE, up front, off the marker our - // worker carries — a name alone cannot tell our container from an operator's, and - // value-equality cannot either (a legitimate re-injection changes the endpoint or - // L1 size). Everything below keys reuse-vs-reject on this. - owned := sglangWireIsOurs(pod) - - // The config mount path is ADAPTER-OWNED (the worker writes the MP config file - // there and the engine reads it). A FOREIGN mount already at that path can - // neither be duplicated (a duplicate mountPath is an invalid Pod) nor safely - // reused (an operator's ConfigMap/secret mount is read-only, so the worker's - // write would fail at runtime) — reject with a message that names the fix; the - // pod webhook turns that into a fail-open admit, so the pod starts un-wired - // rather than broken. This differs from /dev/shm, which IS reused because it is - // plain shared scratch tmpfs the operator legitimately owns. - if existing := mountAtPath(c.VolumeMounts, sglangConfigMountPath); existing != nil && !sglangMountIsOurs(existing, sglangConfigVolumeName, owned) { - return fmt.Errorf("inject engine config: engine container already mounts %q (volume %q), but that path is reserved for the LMCache MP config file the worker writes — move that mount elsewhere", - sglangConfigMountPath, existing.Name) - } - - if work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ - Name: sglangConfigVolumeName, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, - }, owned); err != nil { - return err - } - - // /dev/shm: GPU engine manifests commonly mount their own tmpfs there already. - // Appending a SECOND mount at the same mountPath makes the Pod INVALID (the API - // server rejects duplicate mountPaths), so reuse the engine's existing volume for - // the worker instead — the MP data path only needs both containers on the SAME - // volume, not on one we created. We add (and size) our own tmpfs only when the - // engine has none (or when the one there is ours, which we re-render so a changed - // l1SizeGB resizes the tmpfs too). Caveat: a reused volume is the operator's, so - // its size is theirs to get right — too small silently degrades L1 to slow pickle - // serde. - shmMount := corev1.VolumeMount{Name: sglangShmVolumeName, MountPath: sglangShmMountPath} - if existing := mountAtPath(c.VolumeMounts, sglangShmMountPath); existing != nil && !sglangMountIsOurs(existing, sglangShmVolumeName, owned) { - // Not every mount can be reused — a read-only or projection-backed one breaks - // the MP data path at runtime, deep inside LMCache. Reject at admission and - // let the webhook fail open. - if err := checkLMCacheMPShmReusable(work.Volumes, *existing); err != nil { - return err - } - // Mirror the engine's subPath. Both containers must land on the SAME - // directory, and "same volume" is not enough: an engine mounting subPath - // "shm" while the worker mounts the volume ROOT gives the two processes - // different directories, which admits cleanly and then silently transfers no - // KV — the worst failure shape available. - shmMount = corev1.VolumeMount{Name: existing.Name, MountPath: sglangShmMountPath, SubPath: existing.SubPath} - } else { - if work.Volumes, err = adoptVolume(work.Volumes, sglangShmVolume(l1SizeGB), owned); err != nil { - return err - } - c.VolumeMounts = upsertMountByName(c.VolumeMounts, shmMount) - } - - worker := sglangMPWorkerContainer(c.Image, c.SecurityContext, cfg, chunkSize, mpPort, l1SizeGB, l2Adapter, shmMount) - if work.InitContainers, err = adoptContainer(work.InitContainers, worker, owned); err != nil { - return err - } - - c.Args = UpsertFlag(c.Args, SGLangEnableLMCacheArg) - c.Args = UpsertFlag(c.Args, SGLangEnableMetricsArg) - c.Args = UpsertArgPair(c.Args, SGLangConfigFileArg, sglangConfigMountPath+"/"+sglangConfigFileName) - c.Env = UpsertEnv(c.Env, corev1.EnvVar{Name: EnvLMCacheUseExperimental, Value: lmcacheUseExperimentalVal}) - c.Env = UpsertEnv(c.Env, corev1.EnvVar{Name: EnvInferenceCacheFailOpen, Value: FailOpenString(cache)}) - c.VolumeMounts = upsertMountByName(c.VolumeMounts, corev1.VolumeMount{Name: sglangConfigVolumeName, MountPath: sglangConfigMountPath}) - - *pod = *work // commit: every guard passed - return nil -} - -// sglangMountIsOurs reports whether an existing mount is one THIS adapter placed — -// i.e. the pod is one we already wired (owned, per [sglangWireIsOurs]) AND the mount -// names the volume we render for that path. Our own mount is a re-injection to -// converge, not a collision to reject. -func sglangMountIsOurs(m *corev1.VolumeMount, volumeName string, owned bool) bool { - return owned && m.Name == volumeName -} - -// mountAtPath returns the existing mount at mountPath, or nil. Two mounts sharing a -// mountPath make the Pod invalid, so callers reuse the existing volume rather than -// appending their own. -func mountAtPath(ms []corev1.VolumeMount, path string) *corev1.VolumeMount { - for i := range ms { - if ms[i].MountPath == path { - return &ms[i] - } - } - return nil -} - -// sglangMPWorkerContainer builds the node-local MP-worker native sidecar. It -// writes the engine's config file (its own mp_host/mp_port) then execs the MP -// server — both in this container so, gated by the startupProbe, the config exists -// and the server listens before the engine starts. The worker image defaults to -// the engine image (guaranteeing the same lmcache version — the two speak the MP -// wire) and is overridable via lmCache.workerImage (or legacy -// spec.lmCache.workerImage). -func sglangMPWorkerContainer(engineImage string, engineSC *corev1.SecurityContext, cfg map[string]string, chunkSize, mpPort, l1SizeGB, l2Adapter string, shmMount corev1.VolumeMount) corev1.Container { - image := ConfigOr(cfg, cfgKeyWorkerImage, engineImage) - configPath := sglangConfigMountPath + "/" + sglangConfigFileName - // The validated invocation is `python3 -m lmcache.v1.multiprocess.server` - // (the documented `lmcache server` CLI is the equivalent entrypoint). mp_host - // is 127.0.0.1 — the worker shares the engine pod's network namespace. - script := fmt.Sprintf( - "set -e; printf 'chunk_size: %s\\nmp_host: \"127.0.0.1\"\\nmp_port: %s\\n' > %s; "+ - "exec python3 -m lmcache.v1.multiprocess.server --host 127.0.0.1 --port %s "+ - "--chunk-size %s --l1-size-gb %s --eviction-policy LRU", - chunkSize, mpPort, configPath, mpPort, chunkSize, l1SizeGB) - if l2Adapter != "" { - script += " --l2-adapter " + shellSingleQuote(l2Adapter) - } - - // The worker holds the L1 in a memory-backed tmpfs charged to its own cgroup, so - // it MUST carry a matching memory request+limit — otherwise the L1 is invisible - // to the scheduler and can overcommit the node into a node-pressure OOM. - var resources corev1.ResourceRequirements - if q, ok := sglangMemBudget(l1SizeGB); ok { - resources = corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceMemory: q}, - Limits: corev1.ResourceList{corev1.ResourceMemory: q}, - } - } - - always := corev1.ContainerRestartPolicyAlways - return corev1.Container{ - Name: sglangMPWorkerContainerName, - Image: image, - RestartPolicy: &always, // native sidecar: starts + gates ready before the engine - Command: []string{"sh", "-c"}, - Args: []string{script}, - Resources: resources, - Env: []corev1.EnvVar{ - // The GPU-less sidecar must SEE the engine's GPU to CUDA-IPC its KV, and - // this is the only mechanism that grants it. GPU-VALIDATED, including the - // negative: with visibility revoked the worker dies on - // - // RuntimeError: Device UUID not found in the discovered devices. - // Please make sure the process can see all the accelerator devices - // - // and the engine never reaches ready. The engine hands the worker a device - // UUID; LMCache's ipc_wrapper resolves it to a local index, which only - // works if the device is visible here. - // - // SCOPING THIS TO THE ENGINE'S DEVICE IS NOT POSSIBLE AT ADMISSION: the - // device plugin assigns the UUID at kubelet time, after this mutation runs - // — there is nothing to narrow to yet. Requesting nvidia.com/gpu for the - // worker would be worse: it burns a second GPU and the scheduler would hand - // it a DIFFERENT device than the engine's. - // - // The isolation cost is real and documented for operators (see the - // GPU-visibility note in docs/design/cachebackend-api.md): on a shared node - // the worker can see every GPU, not just its engine's. Note this is the - // engine image's own posture, not something this adapter introduces — - // sglang images ship NVIDIA_VISIBLE_DEVICES=all in their ENV, and the - // device plugin only overrides it for containers that request a GPU (the - // engine gets a UUID; a request-less sidecar keeps the image default). - // Setting it explicitly keeps the wire working on a workerImage that does - // not carry that default, rather than depending on an image side effect. - {Name: "NVIDIA_VISIBLE_DEVICES", Value: "all"}, - // Marks this container as ours so a re-injection converges it instead of - // mistaking it for an operator's name squat (see sglangWireIsOurs). - {Name: envSGLangMPWorkerManaged, Value: sglangMPWorkerManagedVal}, - }, - // shmMount is the engine's existing /dev/shm mount when it has one — volume - // AND subPath mirrored, so the two containers share the same directory - // without a duplicate mountPath — else our own sized tmpfs. - VolumeMounts: []corev1.VolumeMount{ - {Name: sglangConfigVolumeName, MountPath: sglangConfigMountPath}, - shmMount, - }, - // The MP server binds mp_port on loopback, which a pod-IP tcp/http probe - // cannot reach — so exec a loopback check inside the container. This gates - // the engine's start on the ZMQ server being up. - // - // Gating the engine on the worker is DELIBERATE, and is the accepted - // fail-open boundary for this pair (see "Fail-open semantics" in - // docs/design/sglang-lmcache-mp-mode.md). The MP worker is a REQUIRED, - // co-scheduled component of the serving stack — the out-of-process analog of - // vLLM's in-process LMCache connector — not a remote dependency: SGLang has - // no cacheless fallback while --enable-lmcache is on, so letting the engine - // start before the worker listens makes it hang/abort, which is strictly - // worse than waiting. The failOpen contract is honored at the tier that can - // actually be "unavailable" — the SHARED L2: the worker comes up L1-only when - // Redis is unreachable (GPU-validated), so an L2 outage degrades rather than - // blocks. A worker that cannot start at all is a pod-health / CacheBackend - // Degraded condition, exactly as a broken engine connector would be. - StartupProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - Exec: &corev1.ExecAction{Command: []string{ - "python3", "-c", - fmt.Sprintf("import socket; socket.create_connection(('127.0.0.1',%s),1)", mpPort), - }}, - }, - PeriodSeconds: 3, - FailureThreshold: 40, - }, - // Restricted-compatible securityContext (see lmCacheMPServerSecurityContext). This - // mutation lands BEFORE Pod Security admission, so a worker that did NOT carry - // the container-only Restricted requirements (allowPrivilegeEscalation: false, - // drop ALL capabilities) would get the whole engine pod REJECTED in a - // restricted namespace — the cache plane breaking the engine, the inverse of - // the fail-open contract. Notably NOT added: IPC_LOCK (an earlier revision - // carried it over from the RDMA reference manifests; the MP wire moves KV over - // CUDA-IPC and /dev/shm, not RDMA, so no capability is needed — and an added - // capability is itself a Restricted violation). - SecurityContext: lmCacheMPServerSecurityContext(engineSC), - } -} - -// lmCacheMPServerSecurityContext builds an MP server's securityContext so it -// never turns an admissible engine pod into a Pod-Security-rejected one. -// -// It always sets the two container-only Restricted requirements — these cannot be -// inherited from the pod, so the server must carry them itself: -// - AllowPrivilegeEscalation=false, -// - Capabilities.Drop=[ALL] (the server needs no capabilities; GPU access is via -// device files + /dev/shm, not caps). -// -// It also sets seccompProfile=RuntimeDefault (Restricted-required; harmless, and GPU -// workloads run under it). It deliberately does NOT set RunAsNonRoot / RunAsUser / -// ReadOnlyRootFilesystem to fixed values: the selected LMCache image owns its -// user and writable-path requirements, and forcing a UID or read-only rootfs can -// break CUDA-IPC or image startup. The legacy renderer passes engineSC so its -// same-image worker mirrors the engine identity; the standalone typed renderer -// passes nil and inherits any Pod-level identity instead. -func lmCacheMPServerSecurityContext(engineSC *corev1.SecurityContext) *corev1.SecurityContext { - no := false - sc := &corev1.SecurityContext{ - AllowPrivilegeEscalation: &no, - Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, - SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, - } - if engineSC != nil { - sc.RunAsNonRoot = engineSC.RunAsNonRoot - sc.RunAsUser = engineSC.RunAsUser - sc.RunAsGroup = engineSC.RunAsGroup - } - return sc -} - -// sglangL2AdapterJSON returns the worker's --l2-adapter config: the resp adapter -// pointed at the resolved Redis endpoint (host:port). -// -// The endpoint may come from a controller-managed Redis Service or a canonical -// External Redis binding. Admission keeps both shapes at bare host:port because -// the RESP adapter takes host and numeric port as separate fields. -func sglangL2AdapterJSON(endpoint string) (string, error) { - host, port, ok := splitLMCacheHostPort(strings.TrimSpace(endpoint)) - if !ok || host == "" || port == "" { - return "", fmt.Errorf("inject engine config: endpoint %q is not a host:port for the resp L2 adapter", endpoint) - } - // The port is emitted UNQUOTED (the resp adapter expects an integer), so a - // non-numeric or out-of-range one would render invalid JSON — and the worker - // would then fail to parse its --l2-adapter and never bind the ZMQ port, leaving - // the engine wedged behind the startup probe forever. That is worse than not - // wiring at all, so reject here: the webhook fails open and the pod starts - // un-wired. status.endpoint is controller-built and always numeric today; this - // is the boundary check that keeps it that way. - if n, err := strconv.Atoi(port); err != nil || n < 1 || n > sglangMaxTCPPort { - return "", fmt.Errorf("inject engine config: endpoint %q has port %q, want an integer in 1-%d — the resp L2 adapter takes an integer port", endpoint, port, sglangMaxTCPPort) - } - return fmt.Sprintf(`{"type":"resp","host":%q,"port":%s}`, host, port), nil -} - -// sglangMemBudget returns the memory budget for the L1 tier: l1SizeGB + 1Gi -// headroom. It sizes BOTH the /dev/shm tmpfs AND the worker container's memory -// request/limit: the L1 lives in a memory-backed emptyDir, which is charged to the -// cgroup of the container that writes it, so a worker with no request/limit would -// not inform scheduling and could overcommit the node into a node-pressure OOM — -// the same bounded-memory posture the Redis L2 render takes. l1SizeGB is a -// sanitized in-range integer (see sglangIntInRangeOr), so this always parses. -func sglangMemBudget(l1SizeGB string) (resource.Quantity, bool) { - q, err := resource.ParseQuantity(l1SizeGB + "Gi") - if err != nil { - return resource.Quantity{}, false - } - q.Add(resource.MustParse("1Gi")) - return q, true -} - -// sglangShmVolume returns the /dev/shm tmpfs volume sized from the L1 budget. A -// memory-backed emptyDir must never be left unbounded or it can exhaust node -// memory. -func sglangShmVolume(l1SizeGB string) corev1.Volume { - v := corev1.Volume{ - Name: sglangShmVolumeName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}, - }, - } - if q, ok := sglangMemBudget(l1SizeGB); ok { - v.EmptyDir.SizeLimit = &q - } - return v -} - -// sglangIntInRangeOr returns cfg[key] iff it is an integer in [1, max], else -// fallback. This is a hard sanitization boundary: chunkSize/mpPort/l1SizeGB are -// substituted into the worker's `sh -c` command and into resource sizing, so a -// non-integer — a typo or an injection attempt like "4; rm -rf /" — must never -// reach the shell, AND an out-of-range value (a port > 65535, or an l1SizeGB so -// large that resource.ParseQuantity can't size /dev/shm and leaves it unbounded) -// must be rejected. It falls back to the (in-range integer) default rather than -// failing injection, so a mistyped tunable degrades to the default instead of -// crashing the pod webhook. fallback MUST itself be an in-range positive integer. -func sglangIntInRangeOr(cfg map[string]string, key, fallback string, max int) string { - v := strings.TrimSpace(ConfigOr(cfg, key, "")) - if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= max { - return v - } - return fallback -} - -func effectiveSGLangLMCacheConfig(cache *cachev1alpha1.CacheBackend) map[string]string { - cfg := make(map[string]string, 4) - if cache.Spec.LMCache == nil { - return cfg - } - if cache.Spec.LMCache.ChunkSizeTokens != nil { - cfg[cfgKeyChunkSize] = strconv.FormatInt(int64(*cache.Spec.LMCache.ChunkSizeTokens), 10) - } - if cache.Spec.LMCache.WorkerImage != "" { - cfg[cfgKeyWorkerImage] = cache.Spec.LMCache.WorkerImage - } - if cache.Spec.LMCache.WorkerPort != nil { - cfg[cfgKeyMPPort] = strconv.FormatInt(int64(*cache.Spec.LMCache.WorkerPort), 10) - } - if host := cache.Spec.LMCache.HostMemory; host != nil && host.Capacity != nil && host.Capacity.Value() > 0 { - cfg[cfgKeyL1SizeGB] = strconv.FormatInt(ceilPositiveBytesToGiB(host.Capacity.Value()), 10) - } - return cfg -} - -// shellSingleQuote wraps s in single quotes for safe use in a `sh -c` script, -// escaping any embedded single quotes. The L2 JSON contains double quotes, so -// single-quoting keeps them intact. -func shellSingleQuote(s string) string { - return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" -} - -// sglangWireIsOurs reports whether THIS adapter already wired pod — i.e. the MP -// worker sidecar it carries is one we rendered, identified by the marker env var -// our render always stamps ([envSGLangMPWorkerManaged]). -// -// This is the ownership test the reserved-name guards key on, and it must be -// evaluated BEFORE any mutation. Two weaker tests were considered and rejected: -// the container NAME alone cannot distinguish our sidecar from an operator's name -// squat (that is the very thing being decided), and value-EQUALITY against a fresh -// render mislabels a legitimate re-injection as foreign the moment any input -// changes — a moved status.endpoint or a retuned l1SizeGB renders a different, but -// still ours, container. A pod annotation would be the idiomatic marker, but -// InjectEngineConfig only receives the PodSpec. -// -// The marker is forgeable, and that is an accepted boundary rather than a hole: an -// operator who both names their container lmcache-mp-worker AND stamps it with this -// exact marker has, as far as any PodSpec-scoped check can tell, declared it -// adapter-owned — and gets it converged on our render. Nothing in a PodSpec proves -// provenance. What the marker does buy is the case that actually happens: an -// ACCIDENTAL name collision carries no marker, so it is rejected, not overwritten. -func sglangWireIsOurs(pod *corev1.PodSpec) bool { - for i := range pod.InitContainers { - if pod.InitContainers[i].Name != sglangMPWorkerContainerName { - continue - } - for _, e := range pod.InitContainers[i].Env { - // Value included: a same-named env carrying anything else is not the - // marker our render stamps. - if e.Name == envSGLangMPWorkerManaged && e.Value == sglangMPWorkerManagedVal { - return true - } - } - } - return false -} - -// adoptContainer appends want; when an entry with the same name already exists it -// either converges it to want (owned — our own prior injection, so the current -// render wins) or rejects it (not owned). -// -// A container carrying our RESERVED name that we did NOT render is a FOREIGN -// collision: mutating admission must never silently erase an operator's container, -// so reject and let the pod webhook fail open — the pod admits un-wired rather than -// corrupted. Silently leaving the foreign container in place is NOT an option here: -// the engine is given --lmcache-config-file regardless, so it would block at -// startup on a config file that nothing writes. owned comes from -// [sglangWireIsOurs]. -func adoptContainer(cs []corev1.Container, want corev1.Container, owned bool) ([]corev1.Container, error) { - for i := range cs { - if cs[i].Name != want.Name { - continue - } - if !owned { - return nil, fmt.Errorf("inject engine config: pod already has a container named %q that this adapter did not render; that name is reserved for the LMCache MP native sidecar — rename your container", want.Name) - } - cs[i] = want // our own prior injection — converge on the current render - return cs, nil - } - return append(cs, want), nil -} - -// adoptVolume is the volume analog of [adoptContainer]: append, converge our own -// prior injection, or reject a foreign volume squatting one of our reserved names -// (replacing it could corrupt unrelated mounts or invalidate the pod). -func adoptVolume(vs []corev1.Volume, want corev1.Volume, owned bool) ([]corev1.Volume, error) { - for i := range vs { - if vs[i].Name != want.Name { - continue - } - if !owned { - return nil, fmt.Errorf("inject engine config: pod already has a volume named %q that this adapter did not render; that name is reserved for the LMCache MP wire — rename your volume", want.Name) - } - vs[i] = want // our own prior injection — converge on the current render - return vs, nil - } - return append(vs, want), nil -} - -// checkLMCacheMPShmReusable rejects an engine-owned /dev/shm mount the MP server cannot -// safely share. The engine and the server exchange KV through this volume, so it -// must be WRITABLE and both containers must resolve it to the SAME directory — -// neither of which the kubelet reports back at admission; getting it wrong surfaces -// as a silent no-transfer at runtime, deep inside LMCache. -// -// Read-only comes in two shapes, and both are checked. The MOUNT's readOnly is the -// obvious one; the SOURCE's is the one that bites, because a mount-level -// readOnly:false does NOT override a source-level readOnly:true. So this checks the -// projection sources the kubelet always mounts read-only (configMap / secret / -// downwardAPI / projected) AND every in-tree source carrying its own readOnly flag. -// Sources with no such flag (emptyDir, hostPath, ephemeral, …) are writable, or -// their writability is the operator's to configure, so they pass. -func checkLMCacheMPShmReusable(vs []corev1.Volume, m corev1.VolumeMount) error { - if m.ReadOnly { - return fmt.Errorf("inject engine config: engine container mounts %q read-only (volume %q), but the LMCache MP data path writes there — drop readOnly or mount it elsewhere", lmCacheMPShmMountPath, m.Name) - } - // subPath is mirrorable (the caller copies it onto the worker's mount); - // subPathExpr is NOT: it expands $(VAR) from the mounting CONTAINER's env, and the - // worker's env is not the engine's, so the same expression can resolve to a - // different directory — or fail to expand at all. Silently landing the two - // containers on different directories is exactly the failure this guard exists to - // prevent, so reject rather than guess. - if m.SubPathExpr != "" { - return fmt.Errorf("inject engine config: engine container mounts %q with subPathExpr %q (volume %q); the LMCache MP server cannot reproduce that expansion in its own env — use a literal subPath, or mount %[1]q without it", lmCacheMPShmMountPath, m.SubPathExpr, m.Name) - } - for i := range vs { - if vs[i].Name != m.Name { - continue - } - var kind, why string - const projected = "which the kubelet mounts read-only" - const declared = "declared readOnly at the volume source" - switch src := vs[i].VolumeSource; { - // Projection sources: always read-only, regardless of any flag. - case src.ConfigMap != nil: - kind, why = "configMap", projected - case src.Secret != nil: - kind, why = "secret", projected - case src.DownwardAPI != nil: - kind, why = "downwardAPI", projected - case src.Projected != nil: - kind, why = "projected", projected - // Sources with their own readOnly flag. Every in-tree source that has one is - // listed: a mount-level readOnly:false does NOT override a source-level - // readOnly:true, so checking only VolumeMount.ReadOnly (above) misses these - // and hands the worker an unwritable /dev/shm. - case src.PersistentVolumeClaim != nil && src.PersistentVolumeClaim.ReadOnly: - kind, why = "persistentVolumeClaim", declared - case src.CSI != nil && src.CSI.ReadOnly != nil && *src.CSI.ReadOnly: - kind, why = "csi", declared - case src.NFS != nil && src.NFS.ReadOnly: - kind, why = "nfs", declared - case src.CephFS != nil && src.CephFS.ReadOnly: - kind, why = "cephfs", declared - case src.RBD != nil && src.RBD.ReadOnly: - kind, why = "rbd", declared - case src.ISCSI != nil && src.ISCSI.ReadOnly: - kind, why = "iscsi", declared - case src.AzureFile != nil && src.AzureFile.ReadOnly: - kind, why = "azureFile", declared - case src.AzureDisk != nil && src.AzureDisk.ReadOnly != nil && *src.AzureDisk.ReadOnly: - kind, why = "azureDisk", declared - case src.Quobyte != nil && src.Quobyte.ReadOnly: - kind, why = "quobyte", declared - case src.PortworxVolume != nil && src.PortworxVolume.ReadOnly: - kind, why = "portworxVolume", declared - case src.ScaleIO != nil && src.ScaleIO.ReadOnly: - kind, why = "scaleIO", declared - case src.StorageOS != nil && src.StorageOS.ReadOnly: - kind, why = "storageos", declared - case src.Glusterfs != nil && src.Glusterfs.ReadOnly: - kind, why = "glusterfs", declared - case src.Cinder != nil && src.Cinder.ReadOnly: - kind, why = "cinder", declared - case src.FlexVolume != nil && src.FlexVolume.ReadOnly: - kind, why = "flexVolume", declared - case src.Flocker != nil: - // No readOnly field; falls through as writable. - return nil - case src.GCEPersistentDisk != nil && src.GCEPersistentDisk.ReadOnly: - kind, why = "gcePersistentDisk", declared - case src.AWSElasticBlockStore != nil && src.AWSElasticBlockStore.ReadOnly: - kind, why = "awsElasticBlockStore", declared - case src.FC != nil && src.FC.ReadOnly: - kind, why = "fc", declared - case src.VsphereVolume != nil, src.PhotonPersistentDisk != nil, src.GitRepo != nil, src.Image != nil: - // Either no readOnly field, or (gitRepo/image) not a shared-writable - // scratch shape anyone mounts at /dev/shm. Left as-is rather than guessed at. - return nil - default: - return nil - } - return fmt.Errorf("inject engine config: engine container mounts %q from a %s volume (%q) %s, but the LMCache MP data path writes there — use an emptyDir (medium: Memory) instead", lmCacheMPShmMountPath, kind, m.Name, why) - } - return nil -} - -// upsertMountByName replaces the volume mount with the same Name, or appends it. -func upsertMountByName(ms []corev1.VolumeMount, m corev1.VolumeMount) []corev1.VolumeMount { - for i := range ms { - if ms[i].Name == m.Name { - ms[i] = m - return ms - } - } - return append(ms, m) -} diff --git a/internal/adapters/builtin/runtime/test_helpers_test.go b/internal/adapters/builtin/runtime/test_helpers_test.go new file mode 100644 index 00000000..b46852b9 --- /dev/null +++ b/internal/adapters/builtin/runtime/test_helpers_test.go @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import corev1 "k8s.io/api/core/v1" + +import backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" + +const testLMCacheServerImage = "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +func findVolume(volumes []corev1.Volume, name string) *corev1.Volume { + for i := range volumes { + if volumes[i].Name == name { + return &volumes[i] + } + } + return nil +} + +func containsArg(args []string, want string) bool { + for _, arg := range args { + if arg == want { + return true + } + } + return false +} + +func respBinding(endpoint string) *backendadapter.Binding { + return &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: endpoint} +} + +func lookupEnv(env []corev1.EnvVar, name string) (string, bool) { + for i := range env { + if env[i].Name == name { + return env[i].Value, true + } + } + return "", false +} + +func findInitContainer(containers []corev1.Container, name string) *corev1.Container { + for i := range containers { + if containers[i].Name == name { + return &containers[i] + } + } + return nil +} + +func hasMount(mounts []corev1.VolumeMount, name string) bool { + for i := range mounts { + if mounts[i].Name == name { + return true + } + } + return false +} + +func envHasFieldRef(env []corev1.EnvVar, name, path string) bool { + for i := range env { + ref := env[i].ValueFrom + if env[i].Name == name && ref != nil && ref.FieldRef != nil && ref.FieldRef.FieldPath == path { + return true + } + } + return false +} diff --git a/internal/adapters/builtin/runtime/vllm_lmcache.go b/internal/adapters/builtin/runtime/vllm_lmcache.go deleted file mode 100644 index 93047518..00000000 --- a/internal/adapters/builtin/runtime/vllm_lmcache.go +++ /dev/null @@ -1,213 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package runtime - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - "github.com/cachebox-project/inference-cache/internal/enginebinding" - backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" - adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" -) - -// vLLM-specific kvevent-subscriber wiring. The subscriber image and -// policy-server address defaults live in lmcache_shared.go. -const ( - // vLLM engine convention: the KV-event ZMQ PUB endpoint binds on :5557 by - // default (the reference stack's --kv-events-config sets - // endpoint=tcp://*:5557). Parameterising via the adapter (not hardcoding in - // the webhook) lets SGLang or another engine adapter pick a different port - // without touching the webhook. - vllmDefaultEngineZMQPortStr = "5557" - - // vllmDefaultMetricsPortStr is the port vLLM serves Prometheus /metrics on by - // default (:8000). Fed to --engine-metrics-url so the stats scraper hits the - // right endpoint. - vllmDefaultMetricsPortStr = "8000" - - // subscriberHashScheme is the canonical hash-scheme tag the vLLM subscriber - // carries. Hard-coded for this adapter (vLLM's block-hash scheme is distinct - // from SGLang's, and the cache plane keys on the scheme to keep them from - // collapsing). - vllmSubscriberHashScheme = "vllm" -) - -// vllmLMCacheAdapter wires vLLM engine pods to an LMCache engine cache and an -// optional remote binding resolved independently by a provider adapter. -// InjectEngineConfig adds the --kv-transfer-config arg and LMCACHE_* env vars -// to the vLLM container, merging with what the pod template already carries; -// ObservationSidecar returns the kvevent-subscriber container the webhook -// appends so the engine pod auto-attaches to the policy server. -// -// This adapter wires vLLM+LMCache, including Mooncake remote bindings via the -// mooncakestore:// protocol. SGLang+LMCache shares -// the observation sidecar but uses its own MP engine wire and a Redis provider -// binding rather than the standalone lmcache-server. -type vllmLMCacheAdapter struct { - subscriber SubscriberConfig -} - -// NewVLLMLMCacheAdapter returns the adapter that wires vLLM engine pods to an -// LMCache CacheBackend. -func NewVLLMLMCacheAdapter(subscriber SubscriberConfig) adapterruntime.KVCacheRuntimeAdapter { - return vllmLMCacheAdapter{subscriber: subscriber} -} - -// Supports matches vLLM runtimes against an LMCache CacheBackend. Any other -// (runtime, backend) combination is left for another adapter — a future -// admission validator surfaces unsupported pairs as ErrNoAdapter. -func (vllmLMCacheAdapter) Supports(runtime adapterruntime.RuntimeID, cache *cachev1alpha1.CacheBackend) bool { - if cache == nil { - return false - } - return runtime == adapterruntime.RuntimeVLLM && - cache.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache -} - -// SupportedPairs lets the registry expose this adapter's canonical pair to -// admission error messages so a user who asked for an unsupported pair can -// see what they could have asked for instead. -func (vllmLMCacheAdapter) SupportedPairs() []adapterruntime.SupportedPair { - return []adapterruntime.SupportedPair{{Runtime: adapterruntime.RuntimeVLLM, Backend: cachev1alpha1.CacheBackendTypeLMCache}} -} - -// ReservedArgs returns the leading flag tokens this adapter injects and that -// the LMCache integration cannot function without. The validating webhook -// blocks an spec.integration.engineOverrides entry that tries to override or -// suppress any of these so the operator cannot silently un-wire the connector. -// -// - "--kv-transfer-config" is the LMCache connector configuration the engine -// reads at startup; suppressing it means no LMCache wiring at all. -// -// Other tunables the operator may legitimately want to change (e.g. perf -// connector-tuning knobs are deliberately NOT reserved. -func (vllmLMCacheAdapter) ReservedArgs() []string { - return []string{defaultEngineKVTransferConfigArg} -} - -// EngineContainerName returns [EngineContainerName] — the canonical name the -// vLLM engine container carries on a pod the adapter mutates. The pod -// webhook resolves the override target via this method so admission overrides -// land on the same container [InjectEngineConfig] modified. -func (vllmLMCacheAdapter) EngineContainerName() string { return EngineContainerName } - -// ReservedEnv returns the env var names this adapter injects and that the -// LMCache integration cannot function without: -// -// - LMCACHE_REMOTE_URL is the address of the rendered cache server; an -// override re-points the engine at a different cache than the CR -// resolved to. -// - VLLM_USE_V1 selects the vLLM v1 codepath the LMCache connector targets. -// - INFERENCECACHE_FAIL_OPEN mirrors spec.integration.failOpen onto the -// pod; allowing an override would silently desync the pod from the CR -// contract and from status.failOpen. -// - PYTHONHASHSEED pins the deterministic NONE_HASH that seeds vLLM's -// prefix-cache block-hash chain across the scheduler + TP worker -// processes; an override re-randomizes it under TP>1 and LMCache reload -// silently 0-hits (full recompute, no crash, no error). The failure mode -// is invisible, so the operator must not be able to suppress it. -// -// Tunables (LMCACHE_CHUNK_SIZE / LMCACHE_REMOTE_SERDE / LMCACHE_LOCAL_CPU / -// LMCACHE_MAX_LOCAL_CPU_SIZE) are perf/mode knobs the operator may legitimately -// want to change and are deliberately NOT reserved. -func (vllmLMCacheAdapter) ReservedEnv() []string { - return []string{ - EnvLMCacheRemoteURL, - EnvVLLMUseV1, - EnvInferenceCacheFailOpen, - EnvPythonHashSeed, - } -} - -// InjectEngineConfig adds the LMCache connector arg and LMCACHE_* env to the -// vLLM container in pod from the structured remote-storage binding. -// -// spec.integration.role maps onto LMCache's kv_role in the connector -// config: ReadOnly → kv_consumer, WriteOnly → kv_producer, ReadWrite -// (and unset / unknown) → kv_both. -func (vllmLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) bool { - return binding == nil || - binding.Protocol == backendadapter.ProtocolLMCache || - binding.Protocol == backendadapter.ProtocolMooncakeStore -} - -func (vllmLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { - if cache != nil && cache.Spec.IsEventsOnly() { - return nil - } - if binding == nil { - return InjectVLLMLMCacheHostOnly(pod, cache) - } - switch binding.Protocol { - case backendadapter.ProtocolLMCache: - return InjectVLLMLMCache(pod, binding.Endpoint, cache) - case backendadapter.ProtocolMooncakeStore: - if err := InjectVLLMMooncake(pod, binding.Endpoint, cache); err != nil { - return err - } - injectMooncakeEngineHostNetwork(pod, cache) - return nil - default: - return fmt.Errorf("vLLM LMCache adapter does not support remote binding protocol %q", binding.Protocol) - } -} - -func injectMooncakeEngineHostNetwork(pod *corev1.PodSpec, cache *cachev1alpha1.CacheBackend) { - if enginebinding.EngineHostNetworkRequested(cache) { - pod.HostNetwork = true - pod.DNSPolicy = corev1.DNSClusterFirstWithHostNet - } -} - -// InjectRouterConfig is a no-op for LMCache: the LMCache topology has no -// router component the controller needs to wire. Returning nil keeps the -// interface contract satisfied so a Registry caller can blindly invoke both -// Inject* paths on a per-pod basis without branching on backend type — per -// [adapterruntime.KVCacheRuntimeAdapter.InjectRouterConfig]: "backends without a router -// component should return nil without touching pod." Input validation is -// intentionally skipped so a router-less backend never forces callers to -// special-case it. -func (vllmLMCacheAdapter) InjectRouterConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { - _ = pod - _ = binding - _ = cache - return nil -} - -// ObservationSidecar returns the kvevent-subscriber container the Pod webhook -// appends to a vLLM engine pod so its KV-cache events flow to the policy -// server. It delegates to the shared internal subscriber renderer, pinning the -// vLLM-specific knobs: --hash-scheme=vllm and the vLLM ZMQ PUB port. The -// eviction-forwarding policy (--ignore-block-removed) is mode-dependent and -// computed by the shared builder (suppressed in Offload where the L2 tier -// retains evicted blocks; forwarded in EventsOnly where there is no L2). The -// subscriber shape is identical for every vLLM-engine L2 backend (LMCache, -// Mooncake) because the KV-event stream comes from vLLM itself, not the L2 store. -func (a vllmLMCacheAdapter) ObservationSidecar(cache *cachev1alpha1.CacheBackend, pod *corev1.Pod) (*corev1.Container, error) { - return renderSubscriberSidecar(subscriberSidecarParams{ - Config: a.subscriber, - Cache: cache, - Pod: pod, - HashScheme: vllmSubscriberHashScheme, - EngineZMQPortStr: vllmDefaultEngineZMQPortStr, - EngineMetricsPortStr: vllmDefaultMetricsPortStr, - EngineContainerName: a.EngineContainerName(), - }) -} - -// Package-local aliases to the engine-wire helpers. Kept so the in-place -// unit tests in vllm_lmcache_test.go continue to assert on the wire format -// through the canonical adapter API surface. New tests for the shared wire -// (LMCache, Mooncake, and External all speak the LMCache connector) belong in -// this package alongside vllm_lmcache_wire_test.go. -const defaultEngineKVTransferConfigArg = "--kv-transfer-config" - -var ( - kvTransferConfig = KVTransferConfig - upsertArgPair = UpsertArgPair -) diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go index 064e3274..0a16017a 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go @@ -21,29 +21,34 @@ const ( vllmLMCacheMPConnectorName = "LMCacheMPConnector" vllmLMCacheMPConnectorModulePath = "lmcache.integration.vllm.lmcache_mp_connector" vllmDisableHybridKVCacheArg = "--disable-hybrid-kv-cache-manager" + vllmDefaultMetricsPortStr = "8000" + vllmDefaultEngineZMQPortStr = "5557" + vllmSubscriberHashScheme = "vllm" + kvRoleConsumer = "kv_consumer" + kvRoleProducer = "kv_producer" + kvRoleBoth = "kv_both" ) -// vllmLMCacheMPAdapter is the typed PodLocal vLLM adapter. It embeds the -// legacy adapter only to reuse engine-neutral observation and kernel-check -// providers; selection and engine injection are implemented independently so -// the legacy LMCacheConnectorV1/IP wire cannot leak into the MP path. type vllmLMCacheMPAdapter struct { - vllmLMCacheAdapter + subscriber SubscriberConfig } -// NewVLLMLMCacheMPAdapter returns the explicit typed PodLocal adapter. Register -// it before NewVLLMLMCacheAdapter because both advertise the canonical -// vllm/LMCache pair and the registry selects the first matching adapter. +// NewVLLMLMCacheMPAdapter returns the typed PodLocal vLLM adapter. func NewVLLMLMCacheMPAdapter(subscriber SubscriberConfig) runtimeadapter.KVCacheRuntimeAdapter { - return vllmLMCacheMPAdapter{vllmLMCacheAdapter: vllmLMCacheAdapter{subscriber: subscriber}} + return vllmLMCacheMPAdapter{subscriber: subscriber} +} + +func (vllmLMCacheMPAdapter) SupportedPairs() []runtimeadapter.SupportedPair { + return []runtimeadapter.SupportedPair{{Runtime: runtimeadapter.RuntimeVLLM, Backend: cachev1alpha1.CacheBackendTypeLMCache}} } func (vllmLMCacheMPAdapter) Supports(runtime runtimeadapter.RuntimeID, cache *cachev1alpha1.CacheBackend) bool { - return cache != nil && - runtime == runtimeadapter.RuntimeVLLM && - cache.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && - cache.Spec.LMCache != nil && - cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal + if cache == nil || runtime != runtimeadapter.RuntimeVLLM || + cache.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { + return false + } + return cache.Spec.IsEventsOnly() || + (cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal) } func (vllmLMCacheMPAdapter) SupportsBinding(binding *backendadapter.Binding) bool { @@ -248,15 +253,6 @@ func (vllmLMCacheMPAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *bac engine := &work.Containers[engineIndex] engine.Args = UpsertArgPair(engine.Args, defaultEngineKVTransferConfigArg, configJSON) engine.Args = UpsertFlag(engine.Args, vllmDisableHybridKVCacheArg) - for _, name := range []string{ - EnvLMCacheRemoteURL, - EnvLMCacheRemoteSerde, - EnvLMCacheChunkSize, - EnvLMCacheLocalCPU, - EnvLMCacheMaxLocalCPU, - } { - engine.Env = removeEnv(engine.Env, name) - } engine.Env = removeEnv(engine.Env, EnvPythonHashSeed) engine.Env = append(engine.Env, corev1.EnvVar{Name: EnvPythonHashSeed, Value: defaultPythonHashSeed}) engine.Env = removeEnv(engine.Env, EnvInferenceCacheFailOpen) @@ -274,5 +270,23 @@ func (vllmLMCacheMPAdapter) ReservedEnv() []string { return []string{EnvPythonHashSeed, EnvInferenceCacheFailOpen} } +func (vllmLMCacheMPAdapter) EngineContainerName() string { return EngineContainerName } + +func (vllmLMCacheMPAdapter) InjectRouterConfig(*corev1.PodSpec, *backendadapter.Binding, *cachev1alpha1.CacheBackend) error { + return nil +} + +func (a vllmLMCacheMPAdapter) ObservationSidecar(cache *cachev1alpha1.CacheBackend, pod *corev1.Pod) (*corev1.Container, error) { + return renderSubscriberSidecar(subscriberSidecarParams{ + Config: a.subscriber, + Cache: cache, + Pod: pod, + HashScheme: vllmSubscriberHashScheme, + EngineMetricsPortStr: vllmDefaultMetricsPortStr, + EngineContainerName: a.EngineContainerName(), + EngineZMQPortStr: vllmDefaultEngineZMQPortStr, + }) +} + var _ runtimeadapter.KVCacheRuntimeAdapter = vllmLMCacheMPAdapter{} var _ runtimeadapter.LMCacheMPRuntimeAdapter = vllmLMCacheMPAdapter{} diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go index c9318cf6..02ed4a79 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go @@ -55,10 +55,9 @@ func newVLLMMPEnginePod(args ...string) *corev1.Pod { }}}} } -func TestVLLMLMCacheMPRegistrySelectionDoesNotChangeLegacy(t *testing.T) { +func TestVLLMLMCacheMPRegistrySelection(t *testing.T) { registry := runtimeadapter.NewRegistry() registry.Register(NewVLLMLMCacheMPAdapter(SubscriberConfig{})) - registry.Register(NewVLLMLMCacheAdapter(SubscriberConfig{})) typed, err := registry.Select(runtimeadapter.RuntimeVLLM, newTypedVLLMMPBackend()) if err != nil { @@ -67,14 +66,6 @@ func TestVLLMLMCacheMPRegistrySelectionDoesNotChangeLegacy(t *testing.T) { if _, ok := typed.(vllmLMCacheMPAdapter); !ok { t.Fatalf("typed adapter = %T, want vllmLMCacheMPAdapter", typed) } - - legacy, err := registry.Select(runtimeadapter.RuntimeVLLM, newLMCacheBackend(nil)) - if err != nil { - t.Fatalf("select legacy adapter: %v", err) - } - if _, ok := legacy.(vllmLMCacheAdapter); !ok { - t.Fatalf("legacy adapter = %T, want vllmLMCacheAdapter", legacy) - } } func TestVLLMLMCacheMPReservedSurface(t *testing.T) { @@ -87,6 +78,66 @@ func TestVLLMLMCacheMPReservedSurface(t *testing.T) { } } +func TestEnginePortFromContainer(t *testing.T) { + mk := func(command, args []string, env []corev1.EnvVar) *corev1.Pod { + return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: EngineContainerName, Command: command, Args: args, Env: env, + }}}} + } + tests := []struct { + name string + pod *corev1.Pod + want string + }{ + {name: "space form", pod: mk(nil, []string{"--port", "40000"}, nil), want: "40000"}, + {name: "equals form", pod: mk(nil, []string{"--port=41000"}, nil), want: "41000"}, + {name: "absent", pod: mk(nil, []string{"--model", "m"}, nil)}, + {name: "malformed", pod: mk(nil, []string{"--port", "abc"}, nil)}, + {name: "out of range", pod: mk(nil, []string{"--port", "70000"}, nil)}, + {name: "last valid wins", pod: mk(nil, []string{"--port=30000", "--port", "31000"}, nil), want: "31000"}, + {name: "invalid last keeps prior valid", pod: mk(nil, []string{"--port=33000", "--port", "abc"}, nil), want: "33000"}, + {name: "command and args", pod: mk([]string{"launch", "--port=30000"}, []string{"--port=31000"}, nil), want: "31000"}, + {name: "literal env reference", pod: mk(nil, []string{"--port=$(ENGINE_PORT)"}, []corev1.EnvVar{{Name: "ENGINE_PORT", Value: "42000"}}), want: "42000"}, + {name: "valueFrom is not statically resolvable", pod: mk(nil, []string{"--port=$(ENGINE_PORT)"}, []corev1.EnvVar{{Name: "ENGINE_PORT", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := enginePortFromContainer(tc.pod, EngineContainerName); got != tc.want { + t.Fatalf("enginePortFromContainer() = %q, want %q", got, tc.want) + } + }) + } + if got := enginePortFromContainer(nil, EngineContainerName); got != "" { + t.Fatalf("enginePortFromContainer(nil) = %q, want empty", got) + } +} + +func TestVLLMLMCacheMPObservationSidecarMetricsURL(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) + cache := newTypedVLLMMPBackend() + cache.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "Qwen/Qwen2.5-0.5B-Instruct"} + + for _, tc := range []struct { + name string + args []string + want string + }{ + {name: "default port", args: []string{"--model", "m"}, want: "http://127.0.0.1:8000/metrics"}, + {name: "custom port", args: []string{"--model", "m", "--port", "40000"}, want: "http://127.0.0.1:40000/metrics"}, + } { + t.Run(tc.name, func(t *testing.T) { + pod := newVLLMMPEnginePod(tc.args...) + sidecar, err := adapter.ObservationSidecar(cache, pod) + if err != nil || sidecar == nil { + t.Fatalf("ObservationSidecar() = (%+v, %v)", sidecar, err) + } + if got, ok := testArgValue(sidecar.Args, "--engine-metrics-url"); !ok || got != tc.want { + t.Fatalf("--engine-metrics-url = %q (present=%t), want %q; args=%v", got, ok, tc.want, sidecar.Args) + } + }) + } +} + func TestVLLMLMCacheMPValidateEngineParallelism(t *testing.T) { adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) tests := []struct { @@ -128,8 +179,6 @@ func TestVLLMLMCacheMPKVTransferConfigRoles(t *testing.T) { role cachev1alpha1.CacheBackendIntegrationRole want string }{ - {role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, want: kvRoleConsumer}, - {role: cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, want: kvRoleProducer}, {role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, want: kvRoleBoth}, {role: "", want: kvRoleBoth}, } @@ -163,8 +212,6 @@ func TestVLLMLMCacheMPInjectsCommonServerAndExternalConnector(t *testing.T) { pod := newVLLMMPEnginePod("--model", "meta-llama/Meta-Llama-3-8B-Instruct", "--tensor-parallel-size=2") pod.Spec.Containers[0].Env = []corev1.EnvVar{ {Name: "KEEP_ME", Value: "yes"}, - {Name: EnvLMCacheRemoteURL, Value: "lm://legacy:8200"}, - {Name: EnvLMCacheChunkSize, Value: "128"}, {Name: EnvPythonHashSeed, Value: "random"}, } @@ -195,12 +242,6 @@ func TestVLLMLMCacheMPInjectsCommonServerAndExternalConnector(t *testing.T) { if got, ok := lookupEnv(engine.Env, "KEEP_ME"); !ok || got != "yes" { t.Fatalf("unrelated env was not preserved: %q, %v", got, ok) } - for _, legacy := range []string{EnvLMCacheRemoteURL, EnvLMCacheChunkSize} { - if _, ok := lookupEnv(engine.Env, legacy); ok { - t.Fatalf("legacy env %s survived typed MP injection: %+v", legacy, engine.Env) - } - } - server := findInitContainer(pod.Spec.InitContainers, lmCacheMPServerContainerName) if server == nil { t.Fatalf("common MP server missing: %+v", pod.Spec.InitContainers) diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_test.go deleted file mode 100644 index bdf09d38..00000000 --- a/internal/adapters/builtin/runtime/vllm_lmcache_test.go +++ /dev/null @@ -1,1398 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package runtime - -import ( - "flag" - "fmt" - "io" - "strconv" - "strings" - "testing" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - provideradapter "github.com/cachebox-project/inference-cache/internal/adapters/builtin/storage" - "github.com/cachebox-project/inference-cache/internal/enginebinding" - backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" - runtimeadapter "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" -) - -func newLMCacheBackend(cfg map[string]string) *cachev1alpha1.CacheBackend { - cb := newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "vllm") - cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{} - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - } - if value := cfg["chunkSize"]; value != "" { - parsed, _ := strconv.ParseInt(value, 10, 32) - chunkSize := int32(parsed) - cb.Spec.LMCache.ChunkSizeTokens = &chunkSize - } - cb.Spec.LMCache.RemoteSerde = cfg["remoteSerde"] - if value := cfg["maxLocalCPU"]; value != "" { - capacity := resource.MustParse(value + "Gi") - cb.Spec.LMCache.HostMemory = &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &capacity} - } else if cfg["localCPU"] == "True" { - capacity := resource.MustParse("20Gi") - cb.Spec.LMCache.HostMemory = &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &capacity} - } - cb.Spec.RemoteStorage.LMCacheServer.Image = cfg["serverImage"] - if value := cfg["serverCommand"]; value != "" { - cb.Spec.RemoteStorage.LMCacheServer.Command = strings.Fields(value) - } - if value := cfg["model"]; value != "" { - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: value} - } - return cb -} - -func newCacheBackend(backendType cachev1alpha1.CacheBackendType, engine string) *cachev1alpha1.CacheBackend { - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, - Spec: cachev1alpha1.CacheBackendSpec{Type: backendType}, - } - switch engine { - case "vllm": - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - case "sglang": - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - } - return cb -} - -func lmCacheBinding(endpoint string) *backendadapter.Binding { - return &backendadapter.Binding{Protocol: backendadapter.ProtocolLMCache, Endpoint: endpoint} -} - -// resolveLMCacheServer keeps the provider-rendering assertions independent -// from the runtime adapter now that provider lifecycle is a separate seam. -func resolveLMCacheServer(_ runtimeadapter.KVCacheRuntimeAdapter, cb *cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { - return provideradapter.ResolveLMCacheServer(cb, "lmcache/standalone:v0.4.7") -} - -// resolvePod unwraps the provider renderer for tests that only assert on the -// rendered pod, failing on error or a nil result. -func resolvePod(t *testing.T, a runtimeadapter.KVCacheRuntimeAdapter, cb *cachev1alpha1.CacheBackend) *corev1.PodSpec { - t.Helper() - pod, _, err := resolveLMCacheServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if pod == nil { - t.Fatalf("ResolveCacheServer returned nil pod") - } - return pod -} - -func TestVLLMLMCacheSupports(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - - cases := []struct { - name string - runtime runtimeadapter.RuntimeID - cache *cachev1alpha1.CacheBackend - want bool - }{ - {"vllm+lmcache", runtimeadapter.RuntimeVLLM, newLMCacheBackend(nil), true}, - {"vllm+unsupported", runtimeadapter.RuntimeVLLM, newCacheBackend(cachev1alpha1.CacheBackendType("unsupported"), "vllm"), false}, - {"sglang+lmcache", runtimeadapter.RuntimeSGLang, newLMCacheBackend(nil), false}, - {"reference+lmcache", runtimeadapter.RuntimeID("reference"), newLMCacheBackend(nil), false}, - {"nil cache", runtimeadapter.RuntimeVLLM, nil, false}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if got := a.Supports(tc.runtime, tc.cache); got != tc.want { - t.Fatalf("Supports(%q, %+v) = %v, want %v", tc.runtime, tc.cache, got, tc.want) - } - }) - } -} - -func TestVLLMLMCacheResolveCacheServer(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - - pod, svc, err := resolveLMCacheServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if pod == nil || svc == nil { - t.Fatalf("ResolveCacheServer returned nil pod or svc") - } - - if len(pod.Containers) != 1 { - t.Fatalf("containers = %d, want 1", len(pod.Containers)) - } - c := pod.Containers[0] - if c.Name != "lmcache-server" { - t.Fatalf("container name = %q, want lmcache-server", c.Name) - } - if c.Image != "lmcache/standalone:v0.4.7" { - t.Fatalf("container image = %q, want lmcache/standalone:v0.4.7 default", c.Image) - } - if len(c.Command) != 1 || c.Command[0] != "lmcache_server" { - t.Fatalf("command = %v, want [lmcache_server]", c.Command) - } - wantArgs := []string{"0.0.0.0", "65432", "cpu"} - if len(c.Args) != len(wantArgs) { - t.Fatalf("args = %v, want %v", c.Args, wantArgs) - } - for i, want := range wantArgs { - if c.Args[i] != want { - t.Fatalf("args[%d] = %q, want %q", i, c.Args[i], want) - } - } - if len(c.Ports) != 1 || c.Ports[0].ContainerPort != 65432 { - t.Fatalf("ports = %v, want a single 65432 port", c.Ports) - } - - // Service spec: adapter fills Type + Ports only — ObjectMeta and Selector - // are the reconciler's responsibility (see runtimeadapter.KVCacheRuntimeAdapter docs). - if svc.Spec.Type != corev1.ServiceTypeClusterIP { - t.Fatalf("svc.Spec.Type = %q, want ClusterIP", svc.Spec.Type) - } - if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 65432 { - t.Fatalf("svc.Spec.Ports = %v, want a single 65432 port", svc.Spec.Ports) - } - if svc.Spec.Selector != nil { - t.Fatalf("svc.Spec.Selector = %v, want nil (reconciler owns the selector)", svc.Spec.Selector) - } - if svc.Name != "" || svc.Namespace != "" { - t.Fatalf("svc ObjectMeta = %q/%q, want empty (reconciler owns ObjectMeta)", svc.Namespace, svc.Name) - } -} - -// TestVLLMLMCacheResolveCacheServerStaysPodNetworkAndVirtualIP bounds the blast -// radius of the Mooncake hostNetwork/headless change. LMCache's lm:// server is a -// single endpoint on one port, so it keeps the portable, non-privileged default: -// an overlay pod behind a virtual ClusterIP. hostNetwork is reserved for backends -// whose data plane genuinely cannot work without it. -func TestVLLMLMCacheResolveCacheServerStaysPodNetworkAndVirtualIP(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - pod, svc, err := resolveLMCacheServer(a, newLMCacheBackend(nil)) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if pod.HostNetwork { - t.Fatal("pod.HostNetwork = true; lmcache must stay on the pod network (portable, Pod-Security friendly)") - } - if svc.Spec.ClusterIP == corev1.ClusterIPNone { - t.Fatal("svc.Spec.ClusterIP = None; lmcache must keep a virtual ClusterIP") - } -} - -func TestVLLMLMCacheResolveCacheServerHasReadinessProbe(t *testing.T) { - // Without a readiness probe on the lm:// port, AvailableReplicas (and - // therefore the CacheBackend's Ready condition) can flip True before - // the server is actually serving — making status optimistic. The - // adapter must render a TCP probe targeting the named lmcache port so - // Ready waits on the real accept loop. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - pod, _, err := resolveLMCacheServer(a, newLMCacheBackend(nil)) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - probe := pod.Containers[0].ReadinessProbe - if probe == nil { - t.Fatalf("ReadinessProbe is nil; want a TCP probe so Ready waits on the actual accept loop") - } - if probe.TCPSocket == nil { - t.Fatalf("ReadinessProbe.TCPSocket is nil; want a TCP-socket probe") - } - if probe.TCPSocket.Port.StrVal != "lmcache" { - t.Fatalf("probe targets %q, want named port \"lmcache\"", probe.TCPSocket.Port.StrVal) - } -} - -func TestVLLMLMCacheResolveCacheServerBoundsRawNilResources(t *testing.T) { - // The renderer keeps the 4Gi/8Gi safety bounds even when an object bypasses - // the mutating webhook and reaches the raw-struct path with nil resources. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - pod, _, err := resolveLMCacheServer(a, newLMCacheBackend(nil)) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - resources := pod.Containers[0].Resources - if got := resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("4Gi")) != 0 { - t.Fatalf("requests.memory = %s, want 4Gi fallback", got.String()) - } - if got := resources.Limits[corev1.ResourceMemory]; got.Cmp(resource.MustParse("8Gi")) != 0 { - t.Fatalf("limits.memory = %s, want 8Gi fallback", got.String()) - } -} - -func TestVLLMLMCacheResolveCacheServerHasCPURequestWhenAutoscaled(t *testing.T) { - // A targetCPUUtilizationPercent HPA needs the pod's CPU request - // as the utilization denominator, so without one the autoscaler - // never gets a usable metric. The adapter must therefore declare - // a CPU request on the lmcache-server container when spec.autoscaling - // is set. A completely omitted resource block receives the bounded memory - // fallback; an explicitly supplied limits-only block remains limits-only. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - pod, _, err := resolveLMCacheServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - reqs := pod.Containers[0].Resources.Requests - cpu, hasCPU := reqs[corev1.ResourceCPU] - if !hasCPU || cpu.IsZero() { - t.Fatalf("container Resources.Requests missing a CPU request under autoscaling: %v", reqs) - } - if memory := reqs[corev1.ResourceMemory]; memory.Cmp(resource.MustParse("4Gi")) != 0 { - t.Fatalf("container Resources.Requests[memory] = %s, want 4Gi fallback", memory.String()) - } -} - -func TestVLLMLMCacheResolveCacheServerAutoscalingPreservesLimitsOnlyResources(t *testing.T) { - // Operator-supplied limits-only spec.remoteStorage.lmCacheServer.resources combined with - // autoscaling MUST surface as: limits intact, requests carry only - // the HPA CPU fallback (no synthesised memory request). The - // previous behavior synthesised a 1Gi memory request whenever - // memory was absent under autoscaling, which silently overrode - // the operator's "limit-only" intent — that is the gap this - // test pins shut. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - } - pod := resolvePod(t, a, cb) - got := pod.Containers[0].Resources - - wantLim := resource.MustParse("8Gi") - if mem := got.Limits[corev1.ResourceMemory]; mem.Cmp(wantLim) != 0 { - t.Fatalf("Limits[memory] = %v, want operator-supplied %v", mem.String(), wantLim.String()) - } - if _, hasMem := got.Requests[corev1.ResourceMemory]; hasMem { - t.Fatalf("Requests[memory] = %v, want unset (operator declared limits-only)", got.Requests[corev1.ResourceMemory]) - } - if cpu, hasCPU := got.Requests[corev1.ResourceCPU]; !hasCPU || cpu.IsZero() { - t.Fatalf("Requests[cpu] = %v, want HPA fallback", cpu) - } -} - -func TestVLLMLMCacheResolveCacheServerHonorsProviderResources(t *testing.T) { - // spec.remoteStorage.lmCacheServer.resources is the operator-owned knob for the lmcache-server - // container's Resources. When set the adapter MUST pass it through - // verbatim (modulo the autoscaling CPU fallback covered in a separate - // test) — the CRD-schema default supplies memory limits to every - // CacheBackend so the cache-server pod is bounded by the cgroup limit - // rather than OOM-killed by the kubelet under T2 load. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("4Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - } - pod := resolvePod(t, a, cb) - got := pod.Containers[0].Resources - - wantReqMem := resource.MustParse("4Gi") - if mem := got.Requests[corev1.ResourceMemory]; mem.Cmp(wantReqMem) != 0 { - t.Fatalf("Requests[memory] = %v, want %v", mem.String(), wantReqMem.String()) - } - wantLimMem := resource.MustParse("8Gi") - if mem := got.Limits[corev1.ResourceMemory]; mem.Cmp(wantLimMem) != 0 { - t.Fatalf("Limits[memory] = %v, want %v", mem.String(), wantLimMem.String()) - } - if _, ok := got.Requests[corev1.ResourceCPU]; ok { - t.Fatalf("Requests[cpu] = %v, want unset (no autoscaling, operator did not opt in)", got.Requests[corev1.ResourceCPU]) - } -} - -func TestVLLMLMCacheResolveCacheServerProviderResourcesNotMutated(t *testing.T) { - // The adapter must not mutate the CacheBackend's spec.remoteStorage.lmCacheServer.resources in - // place — controllers reconcile against an informer-cached object, - // and a write through the pointer would propagate back to every - // subsequent reader on the same shared cache. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - } - _ = resolvePod(t, a, cb) - - if cb.Spec.RemoteStorage.LMCacheServer.Resources.Requests != nil { - t.Fatalf("spec.remoteStorage.lmCacheServer.resources.requests = %v, want nil (adapter mutated the spec)", cb.Spec.RemoteStorage.LMCacheServer.Resources.Requests) - } -} - -func TestVLLMLMCacheResolveCacheServerEmptyProviderResourcesIsRespected(t *testing.T) { - // An operator who explicitly supplies `spec.remoteStorage.lmCacheServer.resources: {}` is - // suppressing the CRD-default memory budget. The adapter MUST honor - // the empty struct as "no Resources" rather than synthesising a - // fallback — otherwise the documented suppress-the-default workflow - // silently re-introduces limits the operator deliberately omitted. - // (No autoscaling here either: the autoscaling-fallback test pins - // the orthogonal HPA-CPU behavior.) - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{} - pod := resolvePod(t, a, cb) - got := pod.Containers[0].Resources - if len(got.Requests) != 0 { - t.Fatalf("Requests = %v, want empty when spec.remoteStorage.lmCacheServer.resources is {} (operator suppressed default)", got.Requests) - } - if len(got.Limits) != 0 { - t.Fatalf("Limits = %v, want empty when spec.remoteStorage.lmCacheServer.resources is {} (operator suppressed default)", got.Limits) - } -} - -func TestVLLMLMCacheResolveCacheServerAutoscalingFillsMissingCPU(t *testing.T) { - // When spec.remoteStorage.lmCacheServer.resources is set but omits a CPU request, autoscaling - // must still get a CPU-request denominator filled in by the adapter - // — otherwise the operator's memory-only spec.remoteStorage.lmCacheServer.resources silently - // breaks the HPA metric path. The adapter MUST NOT overwrite a - // CPU request the operator did supply. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("4Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("8Gi"), - }, - } - pod := resolvePod(t, a, cb) - reqs := pod.Containers[0].Resources.Requests - cpu, hasCPU := reqs[corev1.ResourceCPU] - if !hasCPU || cpu.IsZero() { - t.Fatalf("Requests[cpu] = %v, want a non-zero CPU fallback for HPA", cpu) - } - // Operator-supplied memory must survive the autoscaling merge. - wantMem := resource.MustParse("4Gi") - if mem := reqs[corev1.ResourceMemory]; mem.Cmp(wantMem) != 0 { - t.Fatalf("Requests[memory] = %v, want operator-supplied %v", mem.String(), wantMem.String()) - } -} - -func TestVLLMLMCacheResolveCacheServerAutoscalingReplacesZeroCPU(t *testing.T) { - // The admission webhook admits `requests.cpu: "0"` (zero is a - // valid kubelet shape — explicit "no guaranteed minimum"), but - // paired with autoscaling it gives the HPA a zero denominator - // and breaks utilization math. The renderer's fallback contract - // is "the HPA always has a usable CPU denominator under - // autoscaling", so a zero (or otherwise non-positive) - // operator-supplied CPU request MUST be replaced with the 250m - // fallback at render time. A POSITIVE operator-supplied value - // still survives — that case is pinned by - // TestVLLMLMCacheResolveCacheServerAutoscalingRespectsOperatorCPU. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("0")}, - } - pod := resolvePod(t, a, cb) - cpu := pod.Containers[0].Resources.Requests[corev1.ResourceCPU] - if cpu.Sign() <= 0 { - t.Fatalf("Requests[cpu] = %v, want non-zero HPA fallback (operator wrote 0)", cpu) - } -} - -func TestVLLMLMCacheResolveCacheServerAutoscalingRespectsOperatorCPU(t *testing.T) { - // If the operator already supplied a CPU request, the autoscaling - // fallback MUST NOT overwrite it — the operator's value is - // authoritative for HPA-utilization math, and a silent overwrite - // would surprise users tuning the denominator. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("750m"), - }, - } - pod := resolvePod(t, a, cb) - wantCPU := resource.MustParse("750m") - if cpu := pod.Containers[0].Resources.Requests[corev1.ResourceCPU]; cpu.Cmp(wantCPU) != 0 { - t.Fatalf("Requests[cpu] = %v, want operator-supplied %v", cpu.String(), wantCPU.String()) - } -} - -func TestVLLMLMCacheResolveCacheServerImageOverride(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(map[string]string{"serverImage": "registry.example.com/lmcache:pinned"}) - - pod, _, err := resolveLMCacheServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - if got := pod.Containers[0].Image; got != "registry.example.com/lmcache:pinned" { - t.Fatalf("container image = %q, want overridden", got) - } -} - -func TestVLLMLMCacheResolveCacheServerCommandOverride(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(map[string]string{ - "serverCommand": "python3 -m lmcache.v1.multiprocess.server --cpu-buffer-size 60", - }) - - pod, _, err := resolveLMCacheServer(a, cb) - if err != nil { - t.Fatalf("ResolveCacheServer: %v", err) - } - c := pod.Containers[0] - if len(c.Command) != 1 || c.Command[0] != "python3" { - t.Fatalf("command = %v, want [python3]", c.Command) - } - wantArgs := []string{"-m", "lmcache.v1.multiprocess.server", "--cpu-buffer-size", "60"} - if len(c.Args) != len(wantArgs) { - t.Fatalf("args = %v, want %v", c.Args, wantArgs) - } - for i, want := range wantArgs { - if c.Args[i] != want { - t.Fatalf("args[%d] = %q, want %q", i, c.Args[i], want) - } - } -} - -func TestVLLMLMCacheResolveCacheServerNilCache(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - if _, _, err := resolveLMCacheServer(a, nil); err == nil { - t.Fatalf("ResolveCacheServer(nil) returned no error") - } -} - -func TestVLLMLMCacheInjectEngineConfig(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: EngineContainerName, - Args: []string{"--enable-prefix-caching", "--max-model-len", "8192"}, - Env: []corev1.EnvVar{ - {Name: "HF_TOKEN", Value: "secret-token"}, - }, - }, - { - Name: "sidecar", - Env: []corev1.EnvVar{{Name: "SIDECAR_VAR", Value: "untouched"}}, - }, - }, - } - - if err := a.InjectEngineConfig(pod, lmCacheBinding("cache.ns1.svc.cluster.local:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - - engine := pod.Containers[0] - url, ok := lookupEnv(engine.Env, EnvLMCacheRemoteURL) - if !ok || url != "lm://cache.ns1.svc.cluster.local:65432" { - t.Fatalf("%s = (%q, %v), want lm://cache.ns1.svc.cluster.local:65432", EnvLMCacheRemoteURL, url, ok) - } - if v, _ := lookupEnv(engine.Env, EnvLMCacheRemoteSerde); v != "naive" { - t.Fatalf("%s = %q, want naive (CPU-safe default)", EnvLMCacheRemoteSerde, v) - } - if v, _ := lookupEnv(engine.Env, EnvLMCacheChunkSize); v != "256" { - t.Fatalf("%s = %q, want 256", EnvLMCacheChunkSize, v) - } - if v, _ := lookupEnv(engine.Env, EnvLMCacheLocalCPU); v != "False" { - t.Fatalf("%s = %q, want False (remote-only by default)", EnvLMCacheLocalCPU, v) - } - if v, _ := lookupEnv(engine.Env, EnvVLLMUseV1); v != "1" { - t.Fatalf("%s = %q, want 1", EnvVLLMUseV1, v) - } - // PYTHONHASHSEED=0 is a correctness invariant: it pins the deterministic - // NONE_HASH across the scheduler + TP worker processes so LMCache reload - // matches under TP>1 (silent 0-hit recompute otherwise). It must be - // injected with exactly "0". - if v, ok := lookupEnv(engine.Env, EnvPythonHashSeed); !ok || v != "0" { - t.Fatalf("%s = (%q, %v), want 0", EnvPythonHashSeed, v, ok) - } - // Existing env on the engine container is preserved. - if v, _ := lookupEnv(engine.Env, "HF_TOKEN"); v != "secret-token" { - t.Fatalf("HF_TOKEN was clobbered: got %q, want secret-token", v) - } - // Existing args are preserved + the connector arg pair is appended. - if !vllmContainsArg(engine.Args, "--enable-prefix-caching") { - t.Fatalf("--enable-prefix-caching was dropped: %v", engine.Args) - } - wantTransfer := kvTransferConfig(cachev1alpha1.CacheBackendIntegrationRoleReadWrite) - if !vllmContainsArgPair(engine.Args, defaultEngineKVTransferConfigArg, wantTransfer) { - t.Fatalf("connector args missing %s %s: %v", defaultEngineKVTransferConfigArg, wantTransfer, engine.Args) - } - - // Sidecar is not the engine container, so it should be untouched. - sidecar := pod.Containers[1] - if _, ok := lookupEnv(sidecar.Env, EnvLMCacheRemoteURL); ok { - t.Fatalf("sidecar got LMCache env injected; the adapter should target only the engine container") - } - if v, _ := lookupEnv(sidecar.Env, "SIDECAR_VAR"); v != "untouched" { - t.Fatalf("SIDECAR_VAR was clobbered: got %q", v) - } -} - -func TestVLLMLMCacheInjectEngineConfigSingleContainerPodAcceptsAnyName(t *testing.T) { - // A pod with exactly one container is accepted as the engine even when - // the container is not named "vllm" — there's no sidecar to crash. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine"}}} - - if err := a.InjectEngineConfig(pod, lmCacheBinding("cache.ns1.svc.cluster.local:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - if _, ok := lookupEnv(pod.Containers[0].Env, EnvLMCacheRemoteURL); !ok { - t.Fatalf("single-container pod missing %s; should have been treated as the engine", EnvLMCacheRemoteURL) - } -} - -func TestVLLMLMCacheInjectEngineConfigMultiContainerWithoutVLLMNameErrors(t *testing.T) { - // A multi-container pod with no container named "vllm" must be - // rejected: blindly mutating every container would inject vLLM-only - // flags onto sidecars and crash them. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{ - {Name: "engine", Env: []corev1.EnvVar{{Name: "EXISTING", Value: "x"}}}, - {Name: "sidecar", Env: []corev1.EnvVar{{Name: "SIDECAR_VAR", Value: "untouched"}}}, - }} - - err := a.InjectEngineConfig(pod, lmCacheBinding("cache.ns1.svc.cluster.local:65432"), cb) - if err == nil { - t.Fatalf("expected an error for multi-container pod without a vllm-named container") - } - // Containers must come back untouched — no partial-mutation footprint. - for _, c := range pod.Containers { - if _, ok := lookupEnv(c.Env, EnvLMCacheRemoteURL); ok { - t.Fatalf("container %q got %s injected before the error: %v", c.Name, EnvLMCacheRemoteURL, c.Env) - } - } -} - -func TestVLLMLMCacheInjectEngineConfigIdempotent(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - - if err := a.InjectEngineConfig(pod, lmCacheBinding("first.svc:65432"), cb); err != nil { - t.Fatalf("first InjectEngineConfig: %v", err) - } - if err := a.InjectEngineConfig(pod, lmCacheBinding("second.svc:65432"), cb); err != nil { - t.Fatalf("second InjectEngineConfig: %v", err) - } - - envs := pod.Containers[0].Env - urlMatches := 0 - for _, e := range envs { - if e.Name == EnvLMCacheRemoteURL { - urlMatches++ - if e.Value != "lm://second.svc:65432" { - t.Fatalf("idempotent inject did not update value: got %q", e.Value) - } - } - } - if urlMatches != 1 { - t.Fatalf("expected exactly 1 %s entry after second inject, got %d", EnvLMCacheRemoteURL, urlMatches) - } - - // Args: the connector arg pair must appear exactly once. - wantTransfer := kvTransferConfig(cachev1alpha1.CacheBackendIntegrationRoleReadWrite) - flagCount := 0 - valueCount := 0 - for _, a := range pod.Containers[0].Args { - if a == defaultEngineKVTransferConfigArg { - flagCount++ - } - if a == wantTransfer { - valueCount++ - } - } - if flagCount != 1 || valueCount != 1 { - t.Fatalf("connector arg pair count = (flag %d, value %d), want (1, 1)", flagCount, valueCount) - } -} - -func TestVLLMLMCacheInjectEngineConfigFailOpen(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - trueVal, falseVal := true, false - cases := []struct { - name string - failOpen *bool - want string - }{ - {"default (unset → true)", nil, "true"}, - {"explicit true", &trueVal, "true"}, - {"explicit false", &falseVal, "false"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cb := newLMCacheBackend(nil) - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - FailOpen: tc.failOpen, - } - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - if err := a.InjectEngineConfig(pod, lmCacheBinding("x.svc:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - if v, _ := lookupEnv(pod.Containers[0].Env, EnvInferenceCacheFailOpen); v != tc.want { - t.Fatalf("%s = %q, want %q", EnvInferenceCacheFailOpen, v, tc.want) - } - }) - } -} - -func TestVLLMLMCacheInjectEngineConfigRoleMapping(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cases := []struct { - role cachev1alpha1.CacheBackendIntegrationRole - wantKVRole string - description string - }{ - {cachev1alpha1.CacheBackendIntegrationRoleReadOnly, "kv_consumer", "ReadOnly → kv_consumer"}, - {cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, "kv_producer", "WriteOnly → kv_producer"}, - {cachev1alpha1.CacheBackendIntegrationRoleReadWrite, "kv_both", "ReadWrite → kv_both"}, - {"", "kv_both", "unset → kv_both (default)"}, - } - for _, tc := range cases { - t.Run(tc.description, func(t *testing.T) { - cb := newLMCacheBackend(nil) - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: tc.role, - } - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - if err := a.InjectEngineConfig(pod, lmCacheBinding("x.svc:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - wantValue := fmt.Sprintf(`{"kv_connector":"LMCacheConnectorV1","kv_role":%q}`, tc.wantKVRole) - if !vllmContainsArgPair(pod.Containers[0].Args, defaultEngineKVTransferConfigArg, wantValue) { - t.Fatalf("Args = %v, want pair (%s, %s)", pod.Containers[0].Args, defaultEngineKVTransferConfigArg, wantValue) - } - }) - } -} - -func TestVLLMLMCacheInjectEngineConfigTypedOverrides(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(map[string]string{ - "chunkSize": "512", - "remoteSerde": "cachegen", - "localCPU": "True", - "maxLocalCPU": "40", - }) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - if err := a.InjectEngineConfig(pod, lmCacheBinding("x.svc:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - checks := map[string]string{ - EnvLMCacheChunkSize: "512", - EnvLMCacheRemoteSerde: "cachegen", - EnvLMCacheLocalCPU: "True", - EnvLMCacheMaxLocalCPU: "40", - } - for name, want := range checks { - if v, _ := lookupEnv(pod.Containers[0].Env, name); v != want { - t.Fatalf("%s = %q, want %q (typed LMCache override)", name, v, want) - } - } -} - -func TestVLLMLMCacheHostOnlyEngineConfigUsesTypedConfig(t *testing.T) { - chunkSize := int32(128) - cb := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - LMCache: &cachev1alpha1.LMCacheEngineSpec{ChunkSizeTokens: &chunkSize}, - }, - } - pod := &corev1.PodSpec{Containers: []corev1.Container{{ - Name: EngineContainerName, - Env: []corev1.EnvVar{ - {Name: EnvLMCacheRemoteURL, Value: "lm://stale-provider:8200"}, - {Name: "KEEP_ME", Value: "preserved"}, - }, - }}} - adapter := NewVLLMLMCacheAdapter(SubscriberConfig{}) - if err := adapter.InjectEngineConfig(pod, nil, cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - env := pod.Containers[0].Env - checks := map[string]string{ - EnvLMCacheChunkSize: "128", - EnvLMCacheRemoteSerde: "naive", - EnvLMCacheLocalCPU: "True", - EnvLMCacheMaxLocalCPU: "20", - } - for name, want := range checks { - if got, _ := lookupEnv(env, name); got != want { - t.Fatalf("%s = %q, want %q", name, got, want) - } - } - if _, ok := lookupEnv(env, EnvLMCacheRemoteURL); ok { - t.Fatalf("%s survived host-only injection", EnvLMCacheRemoteURL) - } - if got, ok := lookupEnv(env, "KEEP_ME"); !ok || got != "preserved" { - t.Fatalf("unrelated env was disturbed: KEEP_ME = %q, present=%v", got, ok) - } -} - -func TestVLLMLMCacheCanonicalMooncakeBindingHonorsEngineHostNetwork(t *testing.T) { - for _, optIn := range []bool{false, true} { - t.Run(fmt.Sprintf("opt-in=%t", optIn), func(t *testing.T) { - cb := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - EngineHostNetwork: optIn, - }, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - }} - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - binding := &backendadapter.Binding{ - Protocol: backendadapter.ProtocolMooncakeStore, - Endpoint: "mooncake.engines.svc.cluster.local:50051", - } - adapter := NewVLLMLMCacheAdapter(SubscriberConfig{}) - if err := adapter.InjectEngineConfig(pod, binding, cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - - if pod.HostNetwork != optIn { - t.Fatalf("pod.HostNetwork = %t, want %t", pod.HostNetwork, optIn) - } - wantDNS := corev1.DNSPolicy("") - if optIn { - wantDNS = corev1.DNSClusterFirstWithHostNet - } - if pod.DNSPolicy != wantDNS { - t.Fatalf("pod.DNSPolicy = %q, want %q", pod.DNSPolicy, wantDNS) - } - if got, _ := lookupEnv(pod.Containers[0].Env, EnvLMCacheRemoteURL); got != "mooncakestore://"+binding.Endpoint { - t.Fatalf("%s = %q, want mooncakestore://%s", EnvLMCacheRemoteURL, got, binding.Endpoint) - } - }) - } -} - -func TestVLLMLMCacheInjectEngineConfigPassesThroughLMScheme(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - // A caller that already prefixed lm:// must not produce lm://lm://. - if err := a.InjectEngineConfig(pod, lmCacheBinding("lm://already.scheme:65432"), cb); err != nil { - t.Fatalf("InjectEngineConfig: %v", err) - } - url, _ := lookupEnv(pod.Containers[0].Env, EnvLMCacheRemoteURL) - if url != "lm://already.scheme:65432" { - t.Fatalf("%s = %q, want pass-through (no double prefix)", EnvLMCacheRemoteURL, url) - } - if strings.HasPrefix(url, "lm://lm://") { - t.Fatalf("double lm:// prefix in %q", url) - } -} - -func TestVLLMLMCacheInjectEngineConfigBadInput(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - good := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName}}} - cases := []struct { - name string - fn func() error - }{ - {"nil pod", func() error { return a.InjectEngineConfig(nil, lmCacheBinding("x.svc:65432"), cb) }}, - {"nil cache", func() error { return a.InjectEngineConfig(good, lmCacheBinding("x.svc:65432"), nil) }}, - {"empty endpoint", func() error { return a.InjectEngineConfig(good, lmCacheBinding(""), cb) }}, - {"no containers", func() error { return a.InjectEngineConfig(&corev1.PodSpec{}, lmCacheBinding("x.svc:65432"), cb) }}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if err := tc.fn(); err == nil { - t.Fatalf("expected error for %s, got nil", tc.name) - } - }) - } -} - -func TestVLLMLMCacheInjectRouterConfigIsNoop(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: "router", Env: []corev1.EnvVar{{Name: "EXISTING", Value: "x"}}}}} - if err := a.InjectRouterConfig(pod, lmCacheBinding("x.svc:65432"), cb); err != nil { - t.Fatalf("InjectRouterConfig: %v", err) - } - // LMCache has no router; the pod must come back untouched (existing env kept, - // no LMCache env added). - if len(pod.Containers[0].Env) != 1 || pod.Containers[0].Env[0].Name != "EXISTING" { - t.Fatalf("InjectRouterConfig modified container env: %v", pod.Containers[0].Env) - } -} - -func TestVLLMLMCacheInjectRouterConfigTrulyNoopsOnBadInput(t *testing.T) { - // The runtimeadapter.KVCacheRuntimeAdapter contract says backends without a router - // component should return nil without touching pod. The LMCache adapter - // must honour that even for nil/empty inputs so callers can blindly - // invoke InjectRouterConfig on every adapter without branching. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(nil) - good := &corev1.PodSpec{Containers: []corev1.Container{{Name: "router"}}} - cases := []struct { - name string - fn func() error - }{ - {"nil pod", func() error { return a.InjectRouterConfig(nil, lmCacheBinding("x"), cb) }}, - {"nil cache", func() error { return a.InjectRouterConfig(good, lmCacheBinding("x"), nil) }}, - {"empty endpoint", func() error { return a.InjectRouterConfig(good, lmCacheBinding(""), cb) }}, - {"no containers", func() error { return a.InjectRouterConfig(&corev1.PodSpec{}, lmCacheBinding("x"), cb) }}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if err := tc.fn(); err != nil { - t.Fatalf("InjectRouterConfig %s returned %v, want nil (router-less backend is a no-op)", tc.name, err) - } - }) - } -} - -// TestVLLMLMCacheReservedArgs pins the reserved-arg list — the args the -// validating webhook will hard-reject from spec.integration.engineOverrides. -// Reservation IS the contract: changing what's reserved without also -// adjusting the documented override surface in docs/design/cachebackend-api.md -// is a contract change, so this test is intentionally exact. -func TestVLLMLMCacheReservedArgs(t *testing.T) { - got := vllmLMCacheAdapter{}.ReservedArgs() - want := []string{defaultEngineKVTransferConfigArg} - if !equalStrSlice(got, want) { - t.Fatalf("ReservedArgs = %v, want %v", got, want) - } -} - -// TestVLLMLMCacheReservedEnv pins the reserved-env list. Four env names -// are reserved here (the integration strictly requires them): the resolved -// remote URL, the v1-codepath selector, the spec.integration.failOpen -// mirror, and PYTHONHASHSEED (the deterministic-NONE_HASH correctness -// invariant whose failure is a silent 0-hit reload under TP>1). Known -// tunables (LMCACHE_CHUNK_SIZE / LMCACHE_REMOTE_SERDE / LMCACHE_LOCAL_CPU / -// LMCACHE_MAX_LOCAL_CPU_SIZE) are NOT reserved, and the test also asserts -// they are absent. (--kv-transfer-config is reserved on the args side, -// covered by TestVLLMLMCacheReservedArgs.) -func TestVLLMLMCacheReservedEnv(t *testing.T) { - got := vllmLMCacheAdapter{}.ReservedEnv() - want := []string{EnvLMCacheRemoteURL, EnvVLLMUseV1, EnvInferenceCacheFailOpen, EnvPythonHashSeed} - if !equalStrSlice(got, want) { - t.Fatalf("ReservedEnv = %v, want %v", got, want) - } - // Negative-control: documented tunables MUST NOT appear in the - // reserved set, or admission would block legitimate operator overrides - // the design explicitly supports. - tunable := map[string]bool{ - EnvLMCacheChunkSize: true, - EnvLMCacheRemoteSerde: true, - EnvLMCacheLocalCPU: true, - EnvLMCacheMaxLocalCPU: true, - } - for _, name := range got { - if tunable[name] { - t.Errorf("env %q is documented as tunable and MUST NOT be reserved", name) - } - } -} - -func TestValidateExternalEndpointProviderSchemes(t *testing.T) { - tests := []struct { - name string - provider cachev1alpha1.CacheBackendRemoteStorageProvider - endpoint string - wantErr bool - }{ - {name: "redis bare", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:6379"}, - {name: "redis rejects lm", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "lm://redis.example:6379", wantErr: true}, - {name: "redis rejects named port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:redis", wantErr: true}, - {name: "redis rejects zero port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:0", wantErr: true}, - {name: "redis rejects out-of-range port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:70000", wantErr: true}, - {name: "lmcache bare", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:8200"}, - {name: "lmcache explicit", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "lm://cache.example:8200"}, - {name: "lmcache rejects mooncake", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "mooncakestore://cache.example:50051", wantErr: true}, - {name: "lmcache rejects named port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:not-a-port", wantErr: true}, - {name: "lmcache rejects zero port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:0", wantErr: true}, - {name: "lmcache rejects out-of-range port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:70000", wantErr: true}, - {name: "mooncake bare", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "cache.example:50051"}, - {name: "mooncake explicit", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://cache.example:50051"}, - {name: "mooncake rejects lm", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "lm://cache.example:50051", wantErr: true}, - {name: "mooncake rejects nested scheme", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://lm://cache.example:50051", wantErr: true}, - {name: "mooncake rejects named port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://cache.example:not-a-port", wantErr: true}, - {name: "mooncake rejects zero port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://cache.example:0", wantErr: true}, - {name: "mooncake rejects out-of-range port", provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://cache.example:70000", wantErr: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := backendadapter.ValidateExternalEndpoint(tt.provider, tt.endpoint) - if tt.wantErr && err == nil { - t.Fatalf("ValidateExternalEndpoint(%s, %q) succeeded, want error", tt.provider, tt.endpoint) - } - if !tt.wantErr && err != nil { - t.Fatalf("ValidateExternalEndpoint(%s, %q): %v", tt.provider, tt.endpoint, err) - } - }) - } -} - -// TestVLLMLMCacheEngineContainerName confirms the adapter exposes its -// canonical container name to the pod webhook so the override merge lands on -// the same container [InjectEngineConfig] modified. -func TestVLLMLMCacheEngineContainerName(t *testing.T) { - if got := (vllmLMCacheAdapter{}).EngineContainerName(); got != EngineContainerName { - t.Fatalf("EngineContainerName = %q, want %q", got, EngineContainerName) - } -} - -func TestRegistryResolvesVLLMLMCache(t *testing.T) { - r := runtimeadapter.NewRegistry() - r.Register(NewVLLMLMCacheAdapter(SubscriberConfig{})) - if r.Len() == 0 { - t.Fatalf("registry has no adapters") - } - got, err := r.Select(runtimeadapter.RuntimeVLLM, newLMCacheBackend(nil)) - if err != nil { - t.Fatalf("Select(vllm, LMCache): %v", err) - } - if _, ok := got.(vllmLMCacheAdapter); !ok { - t.Fatalf("Select returned %T, want vllmLMCacheAdapter", got) - } -} - -func TestUpsertArgPairAppendsAndReplaces(t *testing.T) { - // Append when missing. - got := upsertArgPair([]string{"--keep"}, "--flag", "v1") - want := []string{"--keep", "--flag", "v1"} - if !equalStrSlice(got, want) { - t.Fatalf("upsertArgPair append = %v, want %v", got, want) - } - // Replace value when flag already present (two-arg form). - got = upsertArgPair([]string{"--flag", "old", "--other"}, "--flag", "new") - want = []string{"--flag", "new", "--other"} - if !equalStrSlice(got, want) { - t.Fatalf("upsertArgPair replace = %v, want %v", got, want) - } - // Trailing flag with no value: append the value. - got = upsertArgPair([]string{"--flag"}, "--flag", "v") - want = []string{"--flag", "v"} - if !equalStrSlice(got, want) { - t.Fatalf("upsertArgPair trailing-flag = %v, want %v", got, want) - } - // Equals form: a single `--flag=old` entry must be replaced in place - // with the two-arg form, not have a second `--flag new` appended. - got = upsertArgPair([]string{"--flag=old", "--other"}, "--flag", "new") - want = []string{"--flag", "new", "--other"} - if !equalStrSlice(got, want) { - t.Fatalf("upsertArgPair equals-form replace = %v, want %v", got, want) - } - // Idempotence across forms: the equals form gets normalised to the - // two-arg form, and a second upsert collapses to a single entry. - got = upsertArgPair([]string{"--flag=v1"}, "--flag", "v1") - got = upsertArgPair(got, "--flag", "v2") - want = []string{"--flag", "v2"} - if !equalStrSlice(got, want) { - t.Fatalf("upsertArgPair equals-then-two-arg idempotence = %v, want %v", got, want) - } -} - -func TestVLLMLMCacheObservationSidecarShape(t *testing.T) { - // Auto-attach is opt-in: the operator passes the subscriber image via - // the controller flag. SubscriberConfig here mirrors the production - // wiring. Without it ObservationSidecar would return nil (see - // TestVLLMLMCacheObservationSidecarSkipsWithoutImage). - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(map[string]string{"model": "Qwen/Qwen2.5-0.5B-Instruct"}) - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "engine-a", Namespace: "engines"}, - } - - c, err := a.ObservationSidecar(cb, pod) - if err != nil { - t.Fatalf("ObservationSidecar: %v", err) - } - if c == nil { - t.Fatalf("ObservationSidecar returned nil for vLLM+LMCache with a model + image set") - } - if c.Name != enginebinding.SubscriberContainerName { - t.Fatalf("container name = %q, want %q", c.Name, enginebinding.SubscriberContainerName) - } - if c.Image != DefaultSubscriberImage { - t.Fatalf("container image = %q, want %q", c.Image, DefaultSubscriberImage) - } - // Downward-API env vars carry the pod's name/namespace at start time — - // vital because pod.Name is empty at admission for generateName pods. - if !vllmEnvHasFieldRef(c.Env, "POD_NAME", "metadata.name") { - t.Fatalf("env missing POD_NAME via downward API: %v", c.Env) - } - if !vllmEnvHasFieldRef(c.Env, "POD_NAMESPACE", "metadata.namespace") { - t.Fatalf("env missing POD_NAMESPACE via downward API: %v", c.Env) - } - wantArgFragments := []string{ - "--engine-endpoint=tcp://127.0.0.1:5557", - "--server=" + DefaultPolicyServerGRPCAddress, - "--replica-id=$(POD_NAME)", - "--tenant-id=$(POD_NAMESPACE)", - "--model-id=Qwen/Qwen2.5-0.5B-Instruct", - "--hash-scheme=vllm", - // Required for vLLM+LMCache: LMCache is an L2 tier that retains - // blocks after the engine evicts them from GPU. Forwarding vLLM's - // per-block BlockRemoved as PREFIX_EVICTED would drop a routing - // hint the replica can still cheaply serve from L2 — the - // cache-stress 0-PREFIX_MATCH regression. Pinning the arg here - // keeps a future adapter edit from silently re-enabling the - // eviction forward and re-introducing the bug. - "--ignore-block-removed=true", - } - for _, want := range wantArgFragments { - if !vllmContainsArg(c.Args, want) { - t.Fatalf("subscriber args missing %q; args = %v", want, c.Args) - } - } - if c.SecurityContext == nil || c.SecurityContext.RunAsNonRoot == nil || !*c.SecurityContext.RunAsNonRoot { - t.Fatalf("SecurityContext must run non-root; got %+v", c.SecurityContext) - } - if c.SecurityContext.Capabilities == nil || len(c.SecurityContext.Capabilities.Drop) == 0 { - t.Fatalf("SecurityContext must drop ALL capabilities; got %+v", c.SecurityContext.Capabilities) - } -} - -func TestVLLMLMCacheInjectEngineConfigEventsOnlyIsNoOp(t *testing.T) { - // Events-only (tier-1 routing) wires NO KV connector — the engine container - // must be left untouched so a hybrid-attention model's KV-cache manager is - // not disabled — and it requires no endpoint (nothing dials a cache server), - // so a nil binding must NOT error the way the managed path does. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) - cb := newLMCacheBackend(map[string]string{"model": "Qwen/Qwen3.6-27B"}) - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - } - pod := &corev1.PodSpec{Containers: []corev1.Container{{Name: EngineContainerName, Args: []string{"--model", "x"}}}} - - if err := a.InjectEngineConfig(pod, nil, cb); err != nil { - t.Fatalf("events-only InjectEngineConfig must be a no-op with no binding, got error: %v", err) - } - if got := len(pod.Containers[0].Args); got != 2 { - t.Fatalf("events-only must not add engine args; args = %v", pod.Containers[0].Args) - } - if got := len(pod.Containers[0].Env); got != 0 { - t.Fatalf("events-only must not inject connector env; env = %v", pod.Containers[0].Env) - } - if vllmContainsArg(pod.Containers[0].Args, defaultEngineKVTransferConfigArg) { - t.Fatalf("events-only must not inject the KV connector arg; args = %v", pod.Containers[0].Args) - } -} - -func TestVLLMLMCacheObservationSidecarEventsOnlyForwardsEvictions(t *testing.T) { - // Events-only has no L2 tier retaining blocks, so a BlockRemoved genuinely - // means the prefix is gone: the subscriber MUST forward evictions, i.e. the - // suppression flag must be ABSENT (the binary defaults it to false). Pinning - // its absence here keeps a future edit from silently re-suppressing - // evictions in events-only and stranding stale routing hints. - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(map[string]string{"model": "Qwen/Qwen3.6-27B"}) - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - } - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-a", Namespace: "engines"}} - - c, err := a.ObservationSidecar(cb, pod) - if err != nil || c == nil { - t.Fatalf("events-only ObservationSidecar: (%v, %v)", c, err) - } - for _, arg := range c.Args { - if strings.HasPrefix(arg, "--ignore-block-removed") { - t.Fatalf("events-only must omit --ignore-block-removed (forward evictions); found %q in %v", arg, c.Args) - } - } - // The rest of the subscriber wiring is identical to Offload mode. - for _, want := range []string{ - "--engine-endpoint=tcp://127.0.0.1:5557", - "--replica-id=$(POD_NAME)", - "--model-id=Qwen/Qwen3.6-27B", - "--hash-scheme=vllm", - } { - if !vllmContainsArg(c.Args, want) { - t.Fatalf("events-only subscriber missing %q; args = %v", want, c.Args) - } - } -} - -func TestVLLMLMCacheObservationSidecarHonoursOptions(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{ - Image: "registry.example.com/subscriber:pinned", - PolicyServerGRPCAddress: "ic-server.custom-ns.svc.cluster.local:9090", - }) - cb := newLMCacheBackend(map[string]string{"model": "MyOrg/MyModel"}) - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-z", Namespace: "engines"}} - - c, err := a.ObservationSidecar(cb, pod) - if err != nil || c == nil { - t.Fatalf("ObservationSidecar: (%v, %v)", c, err) - } - if c.Image != "registry.example.com/subscriber:pinned" { - t.Fatalf("image override ignored: got %q", c.Image) - } - if !vllmContainsArg(c.Args, "--server=ic-server.custom-ns.svc.cluster.local:9090") { - t.Fatalf("server address override ignored; args = %v", c.Args) - } -} - -func TestVLLMLMCacheObservationSidecarSkipsWithoutModel(t *testing.T) { - // observation.modelID is the source of --model-id. Without it the - // subscriber binary would refuse to start (model-id is a required - // flag), so the adapter returns (nil, nil) to skip the append. The next - // admission picks up the sidecar once the operator sets the field. - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(nil) - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-a"}} - - c, err := a.ObservationSidecar(cb, pod) - if err != nil { - t.Fatalf("ObservationSidecar: %v", err) - } - if c != nil { - t.Fatalf("expected nil sidecar when observation.modelID is unset, got %+v", c) - } -} - -func TestVLLMLMCacheObservationSidecarSkipsWithoutImage(t *testing.T) { - // Default install opts OUT of auto-attach: when the controller flag - // --kvevent-subscriber-image is unset, the adapter returns no sidecar - // at all — even when observation.modelID is set — so an operator that - // hasn't yet shipped a subscriber image can't end up with engine pods - // stuck in ImagePullBackOff. Opt-in by setting SubscriberConfig.Image. - a := NewVLLMLMCacheAdapter(SubscriberConfig{}) // no image configured - cb := newLMCacheBackend(map[string]string{"model": "MyOrg/MyModel"}) - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-a"}} - - c, err := a.ObservationSidecar(cb, pod) - if err != nil { - t.Fatalf("ObservationSidecar: %v", err) - } - if c != nil { - t.Fatalf("expected nil sidecar when subscriber image is unconfigured, got %+v", c) - } -} - -func TestVLLMLMCacheObservationSidecarArgsParseAgainstSubscriberFlagSet(t *testing.T) { - // Regression: the Go flag package exits on unknown flags, so a sidecar - // arg that the kvevent-subscriber binary doesn't recognise crashes the - // container at startup and the engine pod silently fails to report - // cache state. This test parses the rendered args through a FlagSet - // mirroring the subscriber binary's flag surface and asserts they - // parse cleanly. Keep the flag set in sync with - // cmd/kvevent-subscriber/main.go — adding a flag to the sidecar's args - // before the binary learns it is what this guard exists to catch. - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(map[string]string{"model": "Qwen/Qwen2.5-0.5B-Instruct"}) - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-a", Namespace: "engines"}} - - c, err := a.ObservationSidecar(cb, pod) - if err != nil || c == nil { - t.Fatalf("ObservationSidecar: (%v, %v)", c, err) - } - - fs := flag.NewFlagSet("kvevent-subscriber", flag.ContinueOnError) - fs.SetOutput(io.Discard) - // Subset of cmd/kvevent-subscriber/main.go's flag surface (the flags the - // renderer emits). --engine-metrics-url is now wired; the remaining - // stats-path flags (--stats-interval, --engine-cache-size-bytes, …) still - // take binary defaults and are intentionally absent from the rendered args. - fs.String("engine-endpoint", "", "") - fs.String("topic", "", "") - fs.String("server", "", "") - fs.String("replica-id", "", "") - fs.String("model-id", "", "") - fs.String("tenant-id", "", "") - fs.String("hash-scheme", "", "") - fs.String("engine-metrics-url", "", "") - fs.Duration("window", 0, "") - fs.Bool("ignore-block-removed", false, "") - - if err := fs.Parse(c.Args); err != nil { - t.Fatalf("rendered sidecar args rejected by subscriber FlagSet: %v\nargs = %v", err, c.Args) - } - // Belt-and-suspenders: parse a control case that should fail so the - // FlagSet isn't silently accepting unknown flags (rules out the test - // being a tautology if someone passes the wrong FlagSet mode). - if err := fs.Parse(append(c.Args, "--definitely-not-a-real-flag=x")); err == nil { - t.Fatalf("control: FlagSet must reject unknown flag --definitely-not-a-real-flag") - } -} - -func TestEnginePortFromContainer(t *testing.T) { - mk := func(args ...string) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: EngineContainerName, Args: args}, - }}} - } - mkCmd := func(cmd ...string) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: EngineContainerName, Command: cmd}, - }}} - } - mkBoth := func(cmd, args []string) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: EngineContainerName, Command: cmd, Args: args}, - }}} - } - mkArgsEnv := func(env []corev1.EnvVar, args ...string) *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: EngineContainerName, Args: args, Env: env}, - }}} - } - cases := []struct { - name string - pod *corev1.Pod - want string - }{ - {"space form", mk("--model", "m", "--port", "40000"), "40000"}, - {"equals form", mk("--port=41000"), "41000"}, - {"absent", mk("--model", "m"), ""}, - {"malformed", mk("--port", "abc"), ""}, - {"out of range", mk("--port", "70000"), ""}, - {"trailing --port", mk("--port"), ""}, - {"duplicate: last wins", mk("--port=30000", "--port=31000"), "31000"}, - {"duplicate space+equals: last wins", mk("--port", "30000", "--port=32000"), "32000"}, - {"last invalid falls back to prior valid", mk("--port=33000", "--port", "abc"), "33000"}, - {"normalizes accepted form", mk("--port=+31000"), "31000"}, - // --port may live in Command (the entrypoint), not Args. - {"port in command (space)", mkCmd("python", "-m", "launch", "--port", "40000"), "40000"}, - {"port in command (equals)", mkCmd("launch", "--port=42000"), "42000"}, - {"command + args: args wins (later in argv)", mkBoth([]string{"launch", "--port=30000"}, []string{"--port=31000"}), "31000"}, - {"command sets port, args unrelated", mkBoth([]string{"launch", "--port", "43000"}, []string{"--model", "m"}), "43000"}, - // $(VAR) references resolve against the container's literal env. - {"env ref (equals)", mkArgsEnv([]corev1.EnvVar{{Name: "ENGINE_PORT", Value: "40000"}}, "--port=$(ENGINE_PORT)"), "40000"}, - {"env ref (space)", mkArgsEnv([]corev1.EnvVar{{Name: "P", Value: "45000"}}, "--port", "$(P)"), "45000"}, - {"env ref undefined falls back", mkArgsEnv(nil, "--port=$(MISSING)"), ""}, - {"env ref valueFrom-only unresolvable", mkArgsEnv([]corev1.EnvVar{{Name: "SEC", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}}, "--port=$(SEC)"), ""}, - } - for _, c := range cases { - if got := enginePortFromContainer(c.pod, EngineContainerName); got != c.want { - t.Errorf("%s: enginePortFromContainer = %q, want %q", c.name, got, c.want) - } - } - // Single-container pod: EngineContainerIndexNamed falls back to the lone - // container even when the queried name doesn't match (the engine IS the only - // container), so its --port is still derived. - if got := enginePortFromContainer(mk("--port", "50000"), "some-other-name"); got != "50000" { - t.Errorf("single-container fallback should derive --port; got %q", got) - } - // Multi-container pod with no matching engine container: the lookup errors - // rather than guess, so we fall back to the default port. - multi := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: "sidecar-a", Args: []string{"--port", "40000"}}, - {Name: "sidecar-b"}, - }}} - if got := enginePortFromContainer(multi, "nonexistent-container"); got != "" { - t.Errorf("multi-container pod with no matching container must yield \"\"; got %q", got) - } - if got := enginePortFromContainer(nil, EngineContainerName); got != "" { - t.Errorf("nil pod must yield \"\"; got %q", got) - } -} - -func TestVLLMObservationSidecarDerivesMetricsPortFromEngineArgs(t *testing.T) { - // P1: an operator running the engine on a custom --port must be scraped there, - // not the hardcoded default. The --engine-metrics-url derives from the engine - // container's --port. - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(map[string]string{"model": "Qwen/Qwen2.5-0.5B-Instruct"}) - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "engine-a", Namespace: "engines"}, - Spec: corev1.PodSpec{Containers: []corev1.Container{ - {Name: EngineContainerName, Args: []string{"--model", "m", "--port", "40000"}}, - }}, - } - c, err := a.ObservationSidecar(cb, pod) - if err != nil || c == nil { - t.Fatalf("ObservationSidecar: (%v, %v)", c, err) - } - if got, ok := testArgValue(c.Args, "--engine-metrics-url"); !ok || got != "http://127.0.0.1:40000/metrics" { - t.Fatalf("metrics URL must derive from engine --port 40000; got %q (ok=%t), args = %v", got, ok, c.Args) - } -} - -func TestVLLMLMCacheObservationSidecarBadInput(t *testing.T) { - a := NewVLLMLMCacheAdapter(SubscriberConfig{Image: DefaultSubscriberImage}) - cb := newLMCacheBackend(map[string]string{"model": "m"}) - cases := []struct { - name string - cb *cachev1alpha1.CacheBackend - pod *corev1.Pod - }{ - {"nil cache", nil, &corev1.Pod{}}, - {"nil pod", cb, nil}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - if _, err := a.ObservationSidecar(tc.cb, tc.pod); err == nil { - t.Fatalf("expected error for %s", tc.name) - } - }) - } -} - -// vllmEnvHasFieldRef returns true if env contains an entry named name backed by -// a fieldRef whose FieldPath matches the given path. Used to assert the -// downward-API env the subscriber needs to resolve $(POD_NAME) / -// $(POD_NAMESPACE) at container start. -func vllmEnvHasFieldRef(env []corev1.EnvVar, name, path string) bool { - for _, e := range env { - if e.Name == name && e.ValueFrom != nil && e.ValueFrom.FieldRef != nil && e.ValueFrom.FieldRef.FieldPath == path { - return true - } - } - return false -} - -func vllmContainsArg(args []string, want string) bool { - for _, a := range args { - if a == want { - return true - } - } - return false -} - -func vllmContainsArgPair(args []string, flag, value string) bool { - for i, a := range args { - if a == flag && i+1 < len(args) && args[i+1] == value { - return true - } - } - return false -} - -func equalStrSlice(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_wire.go b/internal/adapters/builtin/runtime/vllm_lmcache_wire.go deleted file mode 100644 index a79ec832..00000000 --- a/internal/adapters/builtin/runtime/vllm_lmcache_wire.go +++ /dev/null @@ -1,503 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -// This file holds the engine-side wire format used by built-in runtime -// adapters that front an LMCache-compatible cache. The in-tree -// vLLM+LMCache adapter uses it for LMCacheServer, Mooncake, and externally -// owned remote bindings; future adapters that speak the protocol can share it. -// -// Centralising the wire keeps the adapters from drifting: an external cache -// the operator manages themselves still presents the same lm:// endpoint -// and the engine still parses the same --kv-transfer-config / LMCACHE_* -// env, so the injection logic is identical and only the endpoint source -// differs. The Mooncake binding reuses the same connector wire — vLLM runs -// the LMCache connector pointed at a mooncakestore:// remote store instead -// of an lm:// one — so it differs from the LMCache path in nothing but the -// remote-URL scheme (see [InjectVLLMMooncake]). It lives with the concrete -// adapters and is not part of the public extension contract. -package runtime - -import ( - "fmt" - "strings" - - corev1 "k8s.io/api/core/v1" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// Engine env var names. The cache plane's contract with the engine: an -// engine pod that carries these variables (plus the --kv-transfer-config -// arg below) is wired to an LMCache-compatible cache. -const ( - EnvLMCacheRemoteURL = "LMCACHE_REMOTE_URL" - EnvLMCacheRemoteSerde = "LMCACHE_REMOTE_SERDE" - EnvLMCacheChunkSize = "LMCACHE_CHUNK_SIZE" - EnvLMCacheLocalCPU = "LMCACHE_LOCAL_CPU" - EnvLMCacheMaxLocalCPU = "LMCACHE_MAX_LOCAL_CPU_SIZE" - EnvVLLMUseV1 = "VLLM_USE_V1" - EnvInferenceCacheFailOpen = "INFERENCECACHE_FAIL_OPEN" - // EnvPythonHashSeed pins Python's hash seed so the NONE_HASH that seeds - // vLLM's prefix-cache block-hash chain is deterministic across the - // scheduler and the TP worker processes. Under TP>1 those are separate - // OS processes; with PYTHONHASHSEED unset each derives a different - // NONE_HASH, so the reload lookup's hashes never match the workers' - // stored hashes — LMCache reload silently 0-hits and the engine fully - // recomputes with no crash and no error. A correctness invariant, not a - // tunable. - EnvPythonHashSeed = "PYTHONHASHSEED" -) - -// EngineContainerName is the conventional name of the vLLM container in an -// engine pod. When a pod has no container with this name, a single-container -// pod is treated as the engine; a multi-container pod is rejected — silently -// mutating every container would inject vLLM-only flags onto sidecars and -// crash them. -const EngineContainerName = "vllm" - -// Defaults the engine env carries when the operator does not override them -// through typed LMCache config. The CPU-safe -// LMCACHE_REMOTE_SERDE is "naive"; "cachegen" is faster but pulls in -// CUDA-only codepaths. -const ( - defaultChunkSize = "256" - defaultRemoteSerde = "naive" - defaultLocalCPU = "False" - defaultMaxLocalCPU = "20" - defaultVLLMUseV1 = "1" - // defaultPythonHashSeed = "0" makes every engine process derive the same - // NONE_HASH so LMCache reload matches under TP>1 (see [EnvPythonHashSeed]). - defaultPythonHashSeed = "0" - kvTransferConfigArg = "--kv-transfer-config" - kvRoleConsumer = "kv_consumer" - kvRoleProducer = "kv_producer" - kvRoleBoth = "kv_both" -) - -// InjectVLLMLMCache adds the LMCache connector arg and LMCACHE_* env to the -// vLLM container in pod, given endpoint as the cache server's address. -// endpoint accepts either a bare `host:port` (canonical) or an already- -// prefixed `lm://host:port` ([LMCacheRemoteURL] passes the prefix through -// rather than doubling it); the helper renders both into the same -// LMCACHE_REMOTE_URL=`lm://host:port`. It merges: existing args/env on -// the vLLM container are preserved, repeat injections are idempotent, -// sidecars are left alone. The engine container is identified by -// [EngineContainerName]; a single-container pod is also accepted (the -// lone container is treated as the engine); a multi-container pod with -// no `vllm` container is rejected. -// -// The in-tree vLLM+LMCache adapter calls this for both managed and externally -// owned bindings. The wire shape is identical; only the endpoint source differs -// (controller-resolved Service DNS vs the operator-supplied -// spec.remoteStorage.endpoint). -func InjectVLLMLMCache(pod *corev1.PodSpec, endpoint string, cache *cachev1alpha1.CacheBackend) error { - return injectLMCacheConnector(pod, endpoint, LMCacheRemoteURL(endpoint), cache) -} - -// InjectVLLMLMCacheHostOnly enables LMCache's engine-local host tier without a -// remote URL. No network provider is selected or contacted. -func InjectVLLMLMCacheHostOnly(pod *corev1.PodSpec, cache *cachev1alpha1.CacheBackend) error { - return injectLMCacheConnector(pod, "", "", cache) -} - -// InjectVLLMMooncake wires a vLLM engine to a Mooncake store. Mooncake -// integrates with vLLM as an LMCache *remote backend*: the engine runs the -// exact same LMCache connector (kv_connector=LMCacheConnectorV1) and reads the -// same LMCACHE_* env, the only difference being the remote-store URL scheme — -// mooncakestore://host:port instead of lm://host:port. LMCache parses that -// scheme (its MooncakestoreConnectorAdapter registers "mooncakestore://") and -// connects the engine to the Mooncake master at host:port, so the injected -// wire is byte-identical to [InjectVLLMLMCache] save for the scheme. endpoint -// accepts a bare host:port (canonical — the Mooncake master Service DNS the -// reconciler published into status.endpoint) or an already-prefixed -// mooncakestore://host:port; both render to LMCACHE_REMOTE_URL= -// mooncakestore://host:port. -// -// Static Mooncake transfer-engine tuning (metadata_server, protocol, -// device_name, segment sizes) lives in LMCache's extra_config, which is -// supplied via an engine-side config file (LMCACHE_CONFIG_FILE / -// MOONCAKE_CONFIG_PATH) the operator owns — it is not env-injectable, so this -// helper wires only the controller-resolved master address + the connector, -// and the transfer-engine defaults (P2P-handshake metadata) cover the simplest -// deployment. See docs/design/cachebackend-api.md for the operator-side config. -func InjectVLLMMooncake(pod *corev1.PodSpec, endpoint string, cache *cachev1alpha1.CacheBackend) error { - return injectLMCacheConnector(pod, endpoint, MooncakeStoreRemoteURL(endpoint), cache) -} - -// injectLMCacheConnector is the shared body behind [InjectVLLMLMCache] and -// [InjectVLLMMooncake]: it merges the LMCache connector arg and LMCACHE_* env -// onto the vLLM container in pod, with remoteURL as the already-scheme-prefixed -// LMCACHE_REMOTE_URL value (lm:// for LMCache, mooncakestore:// for Mooncake). -// endpoint is the pre-scheme address, passed only so the input validation -// reports the same "endpoint is empty" error regardless of scheme. It merges: -// existing args/env on the vLLM container are preserved, repeat injections are -// idempotent, sidecars are left alone. The engine container is identified by -// [EngineContainerName]; a single-container pod is also accepted (the lone -// container is treated as the engine); a multi-container pod with no `vllm` -// container is rejected. -func injectLMCacheConnector(pod *corev1.PodSpec, endpoint, remoteURL string, cache *cachev1alpha1.CacheBackend) error { - if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { - return err - } - if remoteURL != "" && endpoint == "" { - return fmt.Errorf("inject engine config: endpoint is empty") - } - env := []corev1.EnvVar{ - {Name: EnvLMCacheRemoteSerde, Value: effectiveRemoteSerde(cache)}, - {Name: EnvLMCacheChunkSize, Value: effectiveChunkSize(cache)}, - {Name: EnvLMCacheLocalCPU, Value: effectiveLocalCPU(cache)}, - {Name: EnvLMCacheMaxLocalCPU, Value: effectiveHostMemoryGB(cache)}, - {Name: EnvVLLMUseV1, Value: defaultVLLMUseV1}, - {Name: EnvInferenceCacheFailOpen, Value: FailOpenString(cache)}, - {Name: EnvPythonHashSeed, Value: defaultPythonHashSeed}, - } - if remoteURL != "" { - env = append([]corev1.EnvVar{{Name: EnvLMCacheRemoteURL, Value: remoteURL}}, env...) - } - args := []string{kvTransferConfigArg, KVTransferConfig(IntegrationRole(cache))} - - i, err := EngineContainerIndex(pod) - if err != nil { - return err - } - for _, e := range env { - pod.Containers[i].Env = UpsertEnv(pod.Containers[i].Env, e) - } - if remoteURL == "" { - pod.Containers[i].Env = removeEnv(pod.Containers[i].Env, EnvLMCacheRemoteURL) - } - pod.Containers[i].Args = UpsertArgPair(pod.Containers[i].Args, args[0], args[1]) - return nil -} - -// UpsertFlag appends the bare boolean flag (e.g. "--enable-lmcache") when -// it is absent, preserving every existing arg; a second call is a no-op. Used -// for store_true flags that carry no value — distinct from [UpsertArgPair], -// which manages a `--flag value` pair. -func UpsertFlag(args []string, flag string) []string { - for _, a := range args { - if a == flag { - return args - } - } - return append(args, flag) -} - -// ValidateInjectInputs centralises the bad-input checks Inject* paths share. -// The role tag flows into the error message so callers can tell which path -// rejected the input ("engine", "router", ...). -func ValidateInjectInputs(pod *corev1.PodSpec, endpoint string, cache *cachev1alpha1.CacheBackend, role string) error { - if err := validateInjectPodCacheInputs(pod, cache, role); err != nil { - return err - } - if endpoint == "" { - return fmt.Errorf("inject %s config: endpoint is empty", role) - } - return nil -} - -func validateInjectPodCacheInputs(pod *corev1.PodSpec, cache *cachev1alpha1.CacheBackend, role string) error { - if pod == nil { - return fmt.Errorf("inject %s config: pod is nil", role) - } - if cache == nil { - return fmt.Errorf("inject %s config: cache is nil", role) - } - if len(pod.Containers) == 0 { - return fmt.Errorf("inject %s config: pod has no containers", role) - } - return nil -} - -func effectiveChunkSize(cache *cachev1alpha1.CacheBackend) string { - if cache.Spec.LMCache != nil && cache.Spec.LMCache.ChunkSizeTokens != nil { - return fmt.Sprintf("%d", *cache.Spec.LMCache.ChunkSizeTokens) - } - return defaultChunkSize -} - -func effectiveRemoteSerde(cache *cachev1alpha1.CacheBackend) string { - if cache.Spec.LMCache != nil && cache.Spec.LMCache.RemoteSerde != "" { - return cache.Spec.LMCache.RemoteSerde - } - return defaultRemoteSerde -} - -func effectiveHostMemoryGB(cache *cachev1alpha1.CacheBackend) string { - if cache.Spec.LMCache != nil && cache.Spec.LMCache.HostMemory != nil && - cache.Spec.LMCache.HostMemory.Capacity != nil { - bytes := cache.Spec.LMCache.HostMemory.Capacity.Value() - if bytes > 0 { - return fmt.Sprintf("%d", ceilPositiveBytesToGiB(bytes)) - } - } - return defaultMaxLocalCPU -} - -func ceilPositiveBytesToGiB(bytes int64) int64 { - const gib = int64(1024 * 1024 * 1024) - gibibytes := bytes / gib - if bytes%gib != 0 { - gibibytes++ - } - return gibibytes -} - -func effectiveLocalCPU(cache *cachev1alpha1.CacheBackend) string { - if cache.Spec.LMCache != nil && cache.Spec.LMCache.HostMemory != nil && - cache.Spec.LMCache.HostMemory.Capacity != nil { - return "True" - } - if cache.Spec.RemoteStorage == nil { - return "True" - } - return defaultLocalCPU -} - -// EngineContainerIndex returns the index of the vLLM engine container the -// adapter should mutate. See [InjectVLLMLMCache] for the selection rules. -func EngineContainerIndex(pod *corev1.PodSpec) (int, error) { - return EngineContainerIndexNamed(pod, EngineContainerName) -} - -// EngineContainerIndexNamed returns the index of the engine container named -// name, falling back to the lone container in a single-container pod (there is -// no sidecar to crash). A multi-container pod with no container named name is -// rejected — blindly mutating every container would inject engine-only flags -// onto unrelated sidecars and crash them. Adapters for engines whose canonical -// container name differs from vLLM's (e.g. SGLang) call this with their own -// name; [EngineContainerIndex] is the vLLM-named convenience wrapper. -func EngineContainerIndexNamed(pod *corev1.PodSpec, name string) (int, error) { - for i := range pod.Containers { - if pod.Containers[i].Name == name { - return i, nil - } - } - if len(pod.Containers) == 1 { - return 0, nil - } - names := make([]string, len(pod.Containers)) - for i := range pod.Containers { - names[i] = pod.Containers[i].Name - } - return -1, fmt.Errorf("inject engine config: pod has %d containers %v but none is named %q; injecting engine flags into unrelated sidecars would crash them — name the engine container %q", - len(pod.Containers), names, name, name) -} - -// LMCacheRemoteURL prefixes an engine-agnostic host:port endpoint with the -// LMCache lm:// scheme. An endpoint already carrying the lm:// scheme is -// normalised to lower-case `lm://` — the admission validator lowercases -// the scheme during shape checks (so `LM://cache.example` admits), and -// without normalisation here that would inject -// `LMCACHE_REMOTE_URL=lm://LM://cache.example`, a double-prefix the engine -// connector rejects. The prefix match is case-insensitive on the scheme -// only; the host portion is preserved verbatim (DNS is case-insensitive -// but rewriting operator-typed casing is not the helper's job). -func LMCacheRemoteURL(endpoint string) string { - return prefixScheme(endpoint, "lm://") -} - -// MooncakeStoreRemoteURL prefixes an engine-agnostic host:port endpoint with -// the LMCache mooncakestore:// remote-store scheme — the Mooncake analog of -// lm://. host:port is the Mooncake master's address (the controller-resolved -// Service DNS the reconciler publishes into status.endpoint); LMCache's -// MooncakestoreConnectorAdapter parses this scheme and connects the engine's -// LMCache connector to the master there. Like [LMCacheRemoteURL] it is -// idempotent — an endpoint already carrying the scheme is normalised to a -// single lower-case `mooncakestore://` prefix rather than doubled — so a -// re-injection produces the same value and the merge stays a no-op. The host -// portion is preserved verbatim (DNS is case-insensitive; rewriting -// operator-typed casing is not this helper's job). -func MooncakeStoreRemoteURL(endpoint string) string { - return prefixScheme(endpoint, "mooncakestore://") -} - -// prefixScheme returns endpoint with scheme guaranteed as a single, lower-case -// prefix: an endpoint that already starts with the scheme (case-insensitively) -// is normalised to the lower-case form rather than double-prefixed; otherwise -// scheme is prepended. Shared by [LMCacheRemoteURL] and -// [MooncakeStoreRemoteURL] so both LMCache-compatible schemes apply the exact -// same idempotent rule. -func prefixScheme(endpoint, scheme string) string { - if len(endpoint) >= len(scheme) && strings.EqualFold(endpoint[:len(scheme)], scheme) { - return scheme + endpoint[len(scheme):] - } - return scheme + endpoint -} - -// FailOpenString returns the bool form of the effective fail-open mode for -// the engine env. The CRD defaults to fail-open via the defaulting webhook; -// the helper handles nil Integration / nil failOpen too — pre-defaulting -// code paths shouldn't crash. -func FailOpenString(cache *cachev1alpha1.CacheBackend) string { - if cachev1alpha1.IntegrationFailOpen(cache.Spec.Integration) { - return "true" - } - return "false" -} - -// IntegrationRole returns the engine's participation role, defaulting to -// ReadWrite (matching the CRD's documented behaviour when integration is -// unset). -func IntegrationRole(cache *cachev1alpha1.CacheBackend) cachev1alpha1.CacheBackendIntegrationRole { - if cache.Spec.Integration == nil || cache.Spec.Integration.Role == "" { - return cachev1alpha1.CacheBackendIntegrationRoleReadWrite - } - return cache.Spec.Integration.Role -} - -// KVTransferConfig renders the --kv-transfer-config JSON for the given role. -// An unrecognised role falls back to kv_both so a future CRD value (added -// after this adapter ships) is not silently dropped from the kv path. -func KVTransferConfig(role cachev1alpha1.CacheBackendIntegrationRole) string { - kvRole := kvRoleBoth - switch role { - case cachev1alpha1.CacheBackendIntegrationRoleReadOnly: - kvRole = kvRoleConsumer - case cachev1alpha1.CacheBackendIntegrationRoleWriteOnly: - kvRole = kvRoleProducer - case cachev1alpha1.CacheBackendIntegrationRoleReadWrite: - kvRole = kvRoleBoth - } - return fmt.Sprintf(`{"kv_connector":"LMCacheConnectorV1","kv_role":%q}`, kvRole) -} - -// UpsertArgPair inserts or updates the flag/value pair `flag value` in args, -// preserving every other arg. Both the two-arg form (`--flag`, `value`) and -// the equals form (`--flag=value`) are recognised: an existing entry in -// either form is updated in place (to the two-arg form), no duplicate is -// appended. A trailing two-arg `--flag` with no value is treated as missing. -// Normalising on the two-arg form keeps the rendered args stable across -// repeat injections so an idempotent reconcile doesn't churn. -func UpsertArgPair(args []string, flag, value string) []string { - prefix := flag + "=" - for i, a := range args { - switch { - case a == flag: - if i+1 < len(args) { - args[i+1] = value - return args - } - return append(args, value) - case strings.HasPrefix(a, prefix): - args[i] = flag - out := make([]string, 0, len(args)+1) - out = append(out, args[:i+1]...) - out = append(out, value) - out = append(out, args[i+1:]...) - return out - } - } - return append(args, flag, value) -} - -// UpsertEnv returns env with want.Name set to want.Value/ValueFrom: updates -// in place if the entry exists, appends otherwise. Used so a second call to -// Inject*Config never produces duplicate env entries and never disturbs -// unrelated ones — the same property real adapters must preserve. -func UpsertEnv(env []corev1.EnvVar, want corev1.EnvVar) []corev1.EnvVar { - for i := range env { - if env[i].Name == want.Name { - env[i].Value = want.Value - env[i].ValueFrom = want.ValueFrom - return env - } - } - return append(env, want) -} - -func removeEnv(env []corev1.EnvVar, name string) []corev1.EnvVar { - out := env[:0] - for _, entry := range env { - if entry.Name != name { - out = append(out, entry) - } - } - return out -} - -// ConfigOr reads key from cfg or returns fallback when key is absent or empty. -func ConfigOr(cfg map[string]string, key, fallback string) string { - if v, ok := cfg[key]; ok && v != "" { - return v - } - return fallback -} - -// ValidateLMCacheEndpoint reports whether s is a usable input to -// [LMCacheRemoteURL] — i.e. whether the resulting LMCACHE_REMOTE_URL the -// engine wire would inject is well-formed. Returns nil on success, or an -// error whose message names the specific shape problem. -// -// The contract surface (must match the validating admission webhook and -// the API/design docs): -// -// - leading/trailing whitespace is trimmed (operator friendliness); -// - allowed shapes: bare `host:port` or explicit `lm://host:port`; -// - host AND port are both required and both non-empty; -// - other URI schemes (`http://`, `https://`, …) are rejected (the -// adapter would otherwise concatenate `lm://` onto the leading scheme -// and produce an unparseable URL); -// - path/query/fragment components are rejected (the LMCache connector -// speaks TCP and would silently drop them); -// - unbracketed IPv6 literals are rejected (`[::1]:8200` is required — -// without brackets the host/port boundary is ambiguous); -// - embedded whitespace or control characters inside the trimmed value -// are rejected (they would inject a malformed LMCACHE_REMOTE_URL the -// engine connector refuses at startup; also defence-in-depth against -// control-char injection into anything that might later template the -// value). -// -// Shared between admission (which wraps the error in a field.Invalid), -// the C2 reconciler (which degrades Ready=False on invalid stored -// values), and the pod-mutating webhook (which fails open on invalid -// stored values so the engine pod admits unwired rather than crashing). -// Centralising the rule here means a future tightening only needs to -// touch one place to ripple to all three layers. -// splitLMCacheHostPort parses a host:port string into its host and port -// halves with bracket-aware IPv6 handling. Returns (host, port, hasPort) -// so callers can tell apart `cache` (no port → hasPort=false) from -// `cache:` (empty port → hasPort=true, port=""). IPv6 literals MUST be -// bracketed (`[::1]:8200`); an unbracketed multi-colon string is -// rejected as malformed. See [ValidateLMCacheEndpoint] for the contract. -func splitLMCacheHostPort(s string) (host, port string, hasPort bool) { - if s == "" { - return "", "", false - } - if strings.HasPrefix(s, "[") { - end := strings.Index(s, "]") - if end <= 1 { - return "", "", false - } - host = s[1:end] - tail := s[end+1:] - if tail == "" { - return host, "", false - } - if !strings.HasPrefix(tail, ":") { - // Unexpected suffix after the bracketed host (e.g. `[::1]junk`). - return "", "", false - } - port = tail[1:] - // The port half cannot itself contain a colon — `[::1]:8200:bad` - // would otherwise pass with port="8200:bad" and inject an - // invalid LMCACHE_REMOTE_URL=lm://[::1]:8200:bad. The bracketed - // form is the canonical shape precisely because it makes the - // host/port boundary unambiguous; reject anything that tries to - // smuggle an extra colon past it. - if strings.Contains(port, ":") { - return "", "", false - } - return host, port, true - } - // Unbracketed multi-colon string is almost certainly an unbracketed - // IPv6 literal — refuse rather than guess at the host/port split. - if strings.Count(s, ":") > 1 { - return "", "", false - } - if i := strings.LastIndex(s, ":"); i >= 0 { - return s[:i], s[i+1:], true - } - return s, "", false -} diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_wire_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_wire_test.go deleted file mode 100644 index 4e0f54b8..00000000 --- a/internal/adapters/builtin/runtime/vllm_lmcache_wire_test.go +++ /dev/null @@ -1,363 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package runtime - -import ( - "math" - "testing" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" -) - -// lookupInjectedEnv returns the value of the env var named name on the engine -// container and whether it was present. -func lookupInjectedEnv(env []corev1.EnvVar, name string) (string, bool) { - for _, e := range env { - if e.Name == name { - return e.Value, true - } - } - return "", false -} - -// TestInjectVLLMLMCache_InjectsEnv pins the full set of env names the engine -// wire injects, and asserts the PYTHONHASHSEED correctness invariant is -// present with value "0". PYTHONHASHSEED pins the deterministic NONE_HASH -// across the scheduler + TP worker processes so LMCache reload matches under -// TP>1 — without it the reload silently 0-hits and the engine fully -// recomputes (no crash, no error). -func TestInjectVLLMLMCache_InjectsEnv(t *testing.T) { - pod := &corev1.PodSpec{ - Containers: []corev1.Container{{Name: EngineContainerName}}, - } - cache := &cachev1alpha1.CacheBackend{} - - if err := InjectVLLMLMCache(pod, "cache.example:65432", cache); err != nil { - t.Fatalf("InjectVLLMLMCache: %v", err) - } - env := pod.Containers[0].Env - - // The exact set of env names the wire injects. Adding/removing one is a - // contract change — this assertion is intentionally exact. - wantNames := map[string]bool{ - EnvLMCacheRemoteURL: true, - EnvLMCacheRemoteSerde: true, - EnvLMCacheChunkSize: true, - EnvLMCacheLocalCPU: true, - EnvLMCacheMaxLocalCPU: true, - EnvVLLMUseV1: true, - EnvInferenceCacheFailOpen: true, - EnvPythonHashSeed: true, - } - gotNames := make(map[string]bool, len(env)) - for _, e := range env { - gotNames[e.Name] = true - } - // Exact count guards against a duplicate injected entry slipping past the - // name-set checks below (the map collapses a duplicate to one key). - if len(env) != len(wantNames) { - t.Errorf("injected env count = %d, want %d (duplicate or missing entry); env = %v", len(env), len(wantNames), env) - } - for name := range wantNames { - if !gotNames[name] { - t.Errorf("injected env missing %q; got %v", name, gotNames) - } - } - for name := range gotNames { - if !wantNames[name] { - t.Errorf("injected env has unexpected entry %q; got %v", name, gotNames) - } - } - - // Focused assertion: the PYTHONHASHSEED correctness invariant is injected - // with exactly "0" so every engine process derives the same NONE_HASH. - if v, ok := lookupInjectedEnv(env, EnvPythonHashSeed); !ok || v != "0" { - t.Fatalf("%s = (%q, %v), want 0", EnvPythonHashSeed, v, ok) - } -} - -func TestEffectiveHostMemoryGBRoundsWithoutOverflow(t *testing.T) { - tests := []struct { - name string - bytes int64 - want string - }{ - {name: "one byte", bytes: 1, want: "1"}, - {name: "exact GiB", bytes: 1024 * 1024 * 1024, want: "1"}, - {name: "one byte over GiB", bytes: 1024*1024*1024 + 1, want: "2"}, - {name: "maximum quantity value", bytes: math.MaxInt64, want: "8589934592"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - capacity := *resource.NewQuantity(tt.bytes, resource.DecimalSI) - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - LMCache: &cachev1alpha1.LMCacheEngineSpec{ - HostMemory: &cachev1alpha1.CacheBackendHostMemorySpec{ - Capacity: &capacity, - }, - }, - }, - } - if got := effectiveHostMemoryGB(cache); got != tt.want { - t.Fatalf("effectiveHostMemoryGB(%d) = %q, want %q", tt.bytes, got, tt.want) - } - }) - } -} - -func TestLMCacheRemoteURL_BareHostGetsScheme(t *testing.T) { - got := LMCacheRemoteURL("cache.example:8200") - want := "lm://cache.example:8200" - if got != want { - t.Fatalf("LMCacheRemoteURL(bare) = %q, want %q", got, want) - } -} - -func TestLMCacheRemoteURL_LowerCaseSchemePreserved(t *testing.T) { - got := LMCacheRemoteURL("lm://cache.example:8200") - want := "lm://cache.example:8200" - if got != want { - t.Fatalf("LMCacheRemoteURL(lower) = %q, want %q", got, want) - } -} - -func TestLMCacheRemoteURL_UpperCaseSchemeNormalised(t *testing.T) { - // Admission lowercases the scheme during validation, so `LM://...` - // admits. The helper must normalise to lower-case `lm://` rather - // than passing through and producing `lm://LM://...` at injection. - cases := []string{ - "LM://cache.example:8200", - "Lm://cache.example:8200", - "lM://cache.example:8200", - } - for _, in := range cases { - got := LMCacheRemoteURL(in) - want := "lm://cache.example:8200" - if got != want { - t.Fatalf("LMCacheRemoteURL(%q) = %q, want %q", in, got, want) - } - } -} - -func TestLMCacheRemoteURL_HostCasingPreserved(t *testing.T) { - // The case normalisation is scoped to the scheme only; the host - // portion is preserved verbatim so we don't silently rewrite the - // operator's typed value. - got := LMCacheRemoteURL("lm://Cache.Example.Com:8200") - want := "lm://Cache.Example.Com:8200" - if got != want { - t.Fatalf("LMCacheRemoteURL(mixed host) = %q, want %q", got, want) - } -} - -func TestLMCacheRemoteURL_ShortInputDoesNotPanic(t *testing.T) { - // Defensive: an input shorter than the scheme length must not - // index out of bounds. (Admission rejects this at the webhook, - // but the helper is part of the engine wire seam and is called - // from an external remote binding without re-validating.) - for _, in := range []string{"", "lm", "lm:", "lm:/"} { - got := LMCacheRemoteURL(in) - want := "lm://" + in - if got != want { - t.Fatalf("LMCacheRemoteURL(%q) = %q, want %q", in, got, want) - } - } -} - -// TestInjectVLLMMooncake_InjectsEnv asserts the Mooncake wire injects the SAME -// env set as the LMCache wire (Mooncake is an LMCache remote backend) but with -// LMCACHE_REMOTE_URL carrying the mooncakestore:// scheme, and the SAME -// LMCacheConnectorV1 --kv-transfer-config arg. The only on-the-wire difference -// from InjectVLLMLMCache is the remote-URL scheme. -func TestInjectVLLMMooncake_InjectsEnv(t *testing.T) { - pod := &corev1.PodSpec{ - Containers: []corev1.Container{{Name: EngineContainerName}}, - } - cache := &cachev1alpha1.CacheBackend{} - - if err := InjectVLLMMooncake(pod, "mooncake.example:50051", cache); err != nil { - t.Fatalf("InjectVLLMMooncake: %v", err) - } - env := pod.Containers[0].Env - - wantNames := map[string]bool{ - EnvLMCacheRemoteURL: true, - EnvLMCacheRemoteSerde: true, - EnvLMCacheChunkSize: true, - EnvLMCacheLocalCPU: true, - EnvLMCacheMaxLocalCPU: true, - EnvVLLMUseV1: true, - EnvInferenceCacheFailOpen: true, - EnvPythonHashSeed: true, - } - gotNames := make(map[string]bool, len(env)) - for _, e := range env { - gotNames[e.Name] = true - } - if len(env) != len(wantNames) { - t.Errorf("injected env count = %d, want %d; env = %v", len(env), len(wantNames), env) - } - for name := range wantNames { - if !gotNames[name] { - t.Errorf("injected env missing %q; got %v", name, gotNames) - } - } - - // The remote URL carries the mooncakestore:// scheme (the defining - // difference from the LMCache wire). - if v, ok := lookupInjectedEnv(env, EnvLMCacheRemoteURL); !ok || v != "mooncakestore://mooncake.example:50051" { - t.Fatalf("%s = (%q, %v), want mooncakestore://mooncake.example:50051", EnvLMCacheRemoteURL, v, ok) - } - // PYTHONHASHSEED correctness invariant still pinned to "0". - if v, ok := lookupInjectedEnv(env, EnvPythonHashSeed); !ok || v != "0" { - t.Fatalf("%s = (%q, %v), want 0", EnvPythonHashSeed, v, ok) - } - // The connector arg is the shared LMCache connector (Mooncake is wired - // as an LMCache remote backend). - args := pod.Containers[0].Args - if len(args) < 2 || args[0] != "--kv-transfer-config" || !contains(args[1], "LMCacheConnectorV1") { - t.Fatalf("kv-transfer-config arg = %v, want LMCacheConnectorV1 connector", args) - } -} - -func TestMooncakeStoreRemoteURL_BareHostGetsScheme(t *testing.T) { - got := MooncakeStoreRemoteURL("mooncake.example:50051") - want := "mooncakestore://mooncake.example:50051" - if got != want { - t.Fatalf("MooncakeStoreRemoteURL(bare) = %q, want %q", got, want) - } -} - -func TestMooncakeStoreRemoteURL_SchemePreserved(t *testing.T) { - got := MooncakeStoreRemoteURL("mooncakestore://mooncake.example:50051") - want := "mooncakestore://mooncake.example:50051" - if got != want { - t.Fatalf("MooncakeStoreRemoteURL(prefixed) = %q, want %q (must not double the scheme)", got, want) - } -} - -func TestMooncakeStoreRemoteURL_UpperCaseSchemeNormalised(t *testing.T) { - for _, in := range []string{ - "MOONCAKESTORE://mooncake.example:50051", - "MooncakeStore://mooncake.example:50051", - } { - got := MooncakeStoreRemoteURL(in) - want := "mooncakestore://mooncake.example:50051" - if got != want { - t.Fatalf("MooncakeStoreRemoteURL(%q) = %q, want %q", in, got, want) - } - } -} - -func TestMooncakeStoreRemoteURL_HostCasingPreserved(t *testing.T) { - got := MooncakeStoreRemoteURL("mooncakestore://Mooncake.Example.Com:50051") - want := "mooncakestore://Mooncake.Example.Com:50051" - if got != want { - t.Fatalf("MooncakeStoreRemoteURL(mixed host) = %q, want %q", got, want) - } -} - -func TestMooncakeStoreRemoteURL_ShortInputDoesNotPanic(t *testing.T) { - for _, in := range []string{"", "moon", "mooncakestore:", "mooncakestore:/"} { - got := MooncakeStoreRemoteURL(in) - want := "mooncakestore://" + in - if got != want { - t.Fatalf("MooncakeStoreRemoteURL(%q) = %q, want %q", in, got, want) - } - } -} - -func TestValidateLMCacheEndpoint(t *testing.T) { - cases := []struct { - name string - input string - wantErr bool - wantMatch string // substring expected in error message; empty = any error - }{ - // Valid shapes. - {name: "bare-host-port", input: "cache.example:8200"}, - {name: "lm-prefixed", input: "lm://cache.example:8200"}, - {name: "lm-prefixed-uppercase", input: "LM://cache.example:8200"}, - {name: "ipv4-host-port", input: "10.0.0.1:8200"}, - {name: "bracketed-ipv6", input: "[2001:db8::1]:8200"}, - {name: "bracketed-ipv6-loopback", input: "[::1]:8200"}, - {name: "minimum-port", input: "cache.example:1"}, - {name: "maximum-port", input: "cache.example:65535"}, - {name: "leading-trailing-whitespace-trimmed", input: " cache.example:8200 "}, - - // Invalid shapes — empty. - {name: "empty", input: "", wantErr: true, wantMatch: "endpoint is empty"}, - {name: "whitespace-only", input: " ", wantErr: true, wantMatch: "endpoint is empty"}, - - // Invalid shapes — schemes. - {name: "https-scheme", input: "https://cache.example:443", wantErr: true, wantMatch: "scheme \"https\" is not supported"}, - {name: "http-scheme", input: "http://cache.example:80", wantErr: true, wantMatch: "scheme \"http\" is not supported"}, - {name: "tcp-scheme", input: "tcp://cache:8200", wantErr: true, wantMatch: "scheme \"tcp\" is not supported"}, - - // Invalid shapes — path/query/fragment. - {name: "lm-with-path", input: "lm://cache:8200/path", wantErr: true, wantMatch: "paths/queries/fragments"}, - {name: "lm-with-query", input: "lm://cache:8200?q=1", wantErr: true, wantMatch: "paths/queries/fragments"}, - {name: "lm-with-fragment", input: "lm://cache:8200#frag", wantErr: true, wantMatch: "paths/queries/fragments"}, - - // Invalid shapes — missing host/port. - {name: "scheme-only", input: "lm://", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "port-only", input: ":8200", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "lm-port-only", input: "lm://:8200", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "host-only-no-port", input: "cache.example", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "trailing-colon-empty-port", input: "cache.example:", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "bracketed-ipv6-no-port", input: "[::1]", wantErr: true, wantMatch: "non-empty host AND port"}, - - // Invalid shapes — port must be a decimal integer in the TCP range. - {name: "named-port", input: "cache.example:not-a-port", wantErr: true, wantMatch: "integer in 1-65535"}, - {name: "signed-port", input: "cache.example:+8200", wantErr: true, wantMatch: "integer in 1-65535"}, - {name: "zero-port", input: "cache.example:0", wantErr: true, wantMatch: "integer in 1-65535"}, - {name: "out-of-range-port", input: "cache.example:70000", wantErr: true, wantMatch: "integer in 1-65535"}, - {name: "ipv6-named-port", input: "[2001:db8::1]:not-a-port", wantErr: true, wantMatch: "integer in 1-65535"}, - - // Invalid shapes — unbracketed IPv6. - {name: "unbracketed-ipv6", input: "2001:db8::1", wantErr: true, wantMatch: "non-empty host AND port"}, - {name: "unbracketed-ipv6-loopback", input: "::1", wantErr: true, wantMatch: "non-empty host AND port"}, - - // Invalid shapes — embedded whitespace/control chars. - {name: "embedded-space-in-host", input: "cache example:8200", wantErr: true, wantMatch: "whitespace or control characters"}, - {name: "embedded-space-in-port", input: "cache:82 00", wantErr: true, wantMatch: "whitespace or control characters"}, - {name: "embedded-tab", input: "cache.example:82\t00", wantErr: true, wantMatch: "whitespace or control characters"}, - {name: "embedded-newline", input: "cache.example:8200\nLMCACHE_LOG_LEVEL=debug", wantErr: true, wantMatch: "whitespace or control characters"}, - {name: "embedded-null", input: "cache.example:82\x0000", wantErr: true, wantMatch: "whitespace or control characters"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - err := backendadapter.ValidateLMCacheEndpoint(tc.input) - if tc.wantErr { - if err == nil { - t.Fatalf("ValidateLMCacheEndpoint(%q) = nil, want error containing %q", tc.input, tc.wantMatch) - } - if tc.wantMatch != "" && !contains(err.Error(), tc.wantMatch) { - t.Fatalf("ValidateLMCacheEndpoint(%q) error = %q, want substring %q", tc.input, err.Error(), tc.wantMatch) - } - return - } - if err != nil { - t.Fatalf("ValidateLMCacheEndpoint(%q) = %v, want nil", tc.input, err) - } - }) - } -} - -func contains(s, sub string) bool { - for i := 0; i+len(sub) <= len(s); i++ { - if s[i:i+len(sub)] == sub { - return true - } - } - return false -} diff --git a/internal/adapters/builtin/storage/effective_config.go b/internal/adapters/builtin/storage/effective_config.go index d50b2f4e..6d215ede 100644 --- a/internal/adapters/builtin/storage/effective_config.go +++ b/internal/adapters/builtin/storage/effective_config.go @@ -28,65 +28,30 @@ func effectiveProviderResources(cache *cachev1alpha1.CacheBackend) *corev1.Resou } storage := cache.Spec.EffectiveRemoteStorage() if storage != nil { - switch storage.Provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: - if storage.Redis != nil && storage.Redis.Resources != nil { - return storage.Redis.Resources - } - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - if storage.LMCacheServer != nil && storage.LMCacheServer.Resources != nil { - return storage.LMCacheServer.Resources - } - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if storage.Mooncake != nil && storage.Mooncake.Resources != nil { - return storage.Mooncake.Resources - } + if storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderRedis && + storage.Redis != nil && storage.Redis.Resources != nil { + return storage.Redis.Resources } } return defaultProviderResources() } -func effectiveProviderImage(cache *cachev1alpha1.CacheBackend, provider cachev1alpha1.CacheBackendRemoteStorageProvider, fallback string) string { - if cache == nil { - return fallback - } - storage := cache.Spec.EffectiveRemoteStorage() - if storage != nil && storage.Provider == provider { - switch provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: - if storage.Redis != nil && storage.Redis.Image != "" { - return storage.Redis.Image - } - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - if storage.LMCacheServer != nil && storage.LMCacheServer.Image != "" { - return storage.LMCacheServer.Image - } - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if storage.Mooncake != nil && storage.Mooncake.Image != "" { - return storage.Mooncake.Image - } - } +func defaultServerResources(cache *cachev1alpha1.CacheBackend) corev1.ResourceRequirements { + if resources := effectiveProviderResources(cache); resources != nil { + return *resources.DeepCopy() } - return fallback + return corev1.ResourceRequirements{} } -func effectiveProviderCommand(cache *cachev1alpha1.CacheBackend, provider cachev1alpha1.CacheBackendRemoteStorageProvider) []string { +func effectiveProviderImage(cache *cachev1alpha1.CacheBackend, provider cachev1alpha1.CacheBackendRemoteStorageProvider, fallback string) string { if cache == nil { - return nil + return fallback } storage := cache.Spec.EffectiveRemoteStorage() - if storage == nil || storage.Provider != provider { - return nil + if storage != nil && storage.Provider == provider && + provider == cachev1alpha1.CacheBackendRemoteStorageProviderRedis && + storage.Redis != nil && storage.Redis.Image != "" { + return storage.Redis.Image } - switch provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - if storage.LMCacheServer != nil { - return storage.LMCacheServer.Command - } - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if storage.Mooncake != nil { - return storage.Mooncake.Command - } - } - return nil + return fallback } diff --git a/internal/adapters/builtin/storage/lmcache_server.go b/internal/adapters/builtin/storage/lmcache_server.go deleted file mode 100644 index 359da924..00000000 --- a/internal/adapters/builtin/storage/lmcache_server.go +++ /dev/null @@ -1,109 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package storage - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/intstr" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// LMCache standalone-server defaults. Resources and command arguments override -// them through remoteStorage.lmCacheServer; the image is supplied by the -// CacheBackend or controller configuration. -const ( - defaultLMCacheServerPort = int32(65432) - defaultLMCacheServerHost = "0.0.0.0" - defaultLMCacheServerStorage = "cpu" - defaultLMCacheServerPortName = "lmcache" -) - -// ResolveLMCacheServer renders the provider-owned standalone LMCache server. -// The reconciler supplies identity, selectors, workload kind, and ownership. -func ResolveLMCacheServer(cache *cachev1alpha1.CacheBackend, controllerImage string) (*corev1.PodSpec, *corev1.Service, error) { - if cache == nil { - return nil, nil, fmt.Errorf("resolve cache server: cache is nil") - } - image := effectiveProviderImage( - cache, - cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - controllerImage, - ) - if image == "" { - return nil, nil, fmt.Errorf("resolve cache server: image is required; set spec.remoteStorage.lmCacheServer.image or controller --lmcache-server-image") - } - - command, args := lmCacheServerCommand() - if typed := effectiveProviderCommand(cache, cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer); len(typed) > 0 { - command, args = typed[:1], typed[1:] - } - container := corev1.Container{ - Name: "lmcache-server", - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: command, - Args: args, - Ports: []corev1.ContainerPort{ - {Name: defaultLMCacheServerPortName, ContainerPort: defaultLMCacheServerPort, Protocol: corev1.ProtocolTCP}, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString(defaultLMCacheServerPortName)}, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 10, - FailureThreshold: 6, - }, - Resources: defaultServerResources(cache), - } - - pod := &corev1.PodSpec{Containers: []corev1.Container{container}} - service := &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - Name: defaultLMCacheServerPortName, - Port: defaultLMCacheServerPort, - TargetPort: intstr.FromString(defaultLMCacheServerPortName), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } - return pod, service, nil -} - -// defaultServerResources returns a deep-copied provider resource block and -// supplies the CPU request required by a CPU-utilization HPA. -func defaultServerResources(cache *cachev1alpha1.CacheBackend) corev1.ResourceRequirements { - var out corev1.ResourceRequirements - if resources := effectiveProviderResources(cache); resources != nil { - out = *resources.DeepCopy() - } - if cache == nil || cache.Spec.Autoscaling == nil { - return out - } - if out.Requests == nil { - out.Requests = corev1.ResourceList{} - } - cpu, hasCPU := out.Requests[corev1.ResourceCPU] - if !hasCPU || cpu.Sign() <= 0 { - out.Requests[corev1.ResourceCPU] = resource.MustParse("250m") - } - return out -} - -func lmCacheServerCommand() (command, args []string) { - return []string{"lmcache_server"}, []string{ - defaultLMCacheServerHost, - fmt.Sprintf("%d", defaultLMCacheServerPort), - defaultLMCacheServerStorage, - } -} diff --git a/internal/adapters/builtin/storage/mooncake.go b/internal/adapters/builtin/storage/mooncake.go deleted file mode 100644 index 4a2cf427..00000000 --- a/internal/adapters/builtin/storage/mooncake.go +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package storage - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// Mooncake provider defaults. Resources override them through the typed -// remoteStorage.mooncake fields. -const ( - // This reference is fully qualified for CRI-O nodes without short-name - // resolution and pinned to the release validated with the matching - // mooncake-transfer-engine client. - // - // TODO(cachebox): digest-pin this image before production. The master - // command and port surface were validated against this tag; do not - // substitute an invented digest. - defaultMooncakeMasterImage = "docker.io/kvcacheai/mooncake:0.3.11.post1" - defaultMooncakeMasterRPCPort = int32(50051) - defaultMooncakeMetadataPort = int32(8080) - defaultMooncakeMetricsPort = int32(9003) - defaultMooncakeMasterHost = "0.0.0.0" - mooncakeRPCPortName = "mooncake-rpc" - mooncakeMetadataPortName = "mooncake-meta" - mooncakeMetricsPortName = "metrics" - mooncakeMasterContainerName = "mooncake-master" -) - -// ResolveMooncakeServer renders the provider-owned Mooncake master workload. -func ResolveMooncakeServer(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { - if cache == nil { - return nil, nil, fmt.Errorf("resolve cache server: cache is nil") - } - image := effectiveProviderImage( - cache, - cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - defaultMooncakeMasterImage, - ) - - command, args := mooncakeMasterCommand() - if typed := effectiveProviderCommand(cache, cachev1alpha1.CacheBackendRemoteStorageProviderMooncake); len(typed) > 0 { - command, args = typed[:1], typed[1:] - } - container := corev1.Container{ - Name: mooncakeMasterContainerName, - Image: image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: command, - Args: args, - Ports: []corev1.ContainerPort{ - {Name: mooncakeRPCPortName, ContainerPort: defaultMooncakeMasterRPCPort, Protocol: corev1.ProtocolTCP}, - {Name: mooncakeMetadataPortName, ContainerPort: defaultMooncakeMetadataPort, Protocol: corev1.ProtocolTCP}, - {Name: mooncakeMetricsPortName, ContainerPort: defaultMooncakeMetricsPort, Protocol: corev1.ProtocolTCP}, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString(mooncakeRPCPortName)}, - }, - InitialDelaySeconds: 5, - PeriodSeconds: 10, - FailureThreshold: 6, - }, - Resources: defaultServerResources(cache), - } - - // Mooncake is a peer-to-peer transfer mesh. The master returns the node - // holding a block, then the engine dials that node on a dynamically - // negotiated port. Host networking plus a headless Service publishes the - // node address without limiting the data path to declared Service ports. - pod := &corev1.PodSpec{ - Containers: []corev1.Container{container}, - HostNetwork: true, - DNSPolicy: corev1.DNSClusterFirstWithHostNet, - } - service := &corev1.Service{ - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - ClusterIP: corev1.ClusterIPNone, - Ports: []corev1.ServicePort{ - { - Name: mooncakeRPCPortName, - Port: defaultMooncakeMasterRPCPort, - TargetPort: intstr.FromString(mooncakeRPCPortName), - Protocol: corev1.ProtocolTCP, - }, - { - Name: mooncakeMetadataPortName, - Port: defaultMooncakeMetadataPort, - TargetPort: intstr.FromString(mooncakeMetadataPortName), - Protocol: corev1.ProtocolTCP, - }, - }, - }, - } - return pod, service, nil -} - -func mooncakeMasterCommand() (command, args []string) { - return []string{"mooncake_master"}, []string{ - fmt.Sprintf("--rpc_port=%d", defaultMooncakeMasterRPCPort), - fmt.Sprintf("--metrics_port=%d", defaultMooncakeMetricsPort), - "--enable_http_metadata_server=true", - "--http_metadata_server_host=" + defaultMooncakeMasterHost, - fmt.Sprintf("--http_metadata_server_port=%d", defaultMooncakeMetadataPort), - } -} diff --git a/internal/adapters/builtin/storage/redis.go b/internal/adapters/builtin/storage/redis.go index 13d80902..c429fc5e 100644 --- a/internal/adapters/builtin/storage/redis.go +++ b/internal/adapters/builtin/storage/redis.go @@ -17,14 +17,9 @@ import ( // // SGLang drives LMCache in multiprocess (MP) mode: the engine attaches to a // node-local MP worker, and the worker offloads its shared/cross-node tier to an -// `--l2-adapter`. Unlike the vLLM `lm://` path, `lm://` is not a valid MP -// `--l2-adapter` type, so the SGLang pair cannot reuse [ResolveLMCacheServer]. -// Redis (the `resp` adapter) is the shared L2: a network-addressable store that -// fits the one-Service, engines-anywhere model exactly (a ClusterIP Service, no -// hostNetwork/mesh — the opposite of Mooncake), it maps onto one Deployment + -// Service, and it is proven end-to-end. The heavier tiers (`s3`, `mooncake_store`) -// are future provider bindings, not the simple default. See -// docs/design/sglang-lmcache-mp-mode.md. +// `--l2-adapter`. Redis (the `resp` adapter) is the shared L2: a +// network-addressable store that fits the one-Service, engines-anywhere model +// exactly. It maps onto one Deployment and Service and is proven end-to-end. // // This render is the shared-store half of the MP data plane; the engine-side wire // (config-file + MP-worker sidecar pointed at this Redis) is injected by the @@ -58,8 +53,7 @@ const ( ) // ResolveRedisL2Server renders the managed Redis L2 store's container set and the -// Service's port set for the SGLang LMCache MP-mode data plane, mirroring the seam -// [ResolveLMCacheServer] uses: the reconciler owns ObjectMeta, the Service +// Service's port set for the SGLang LMCache MP-mode data plane. The reconciler owns ObjectMeta, the Service // Selector, the workload kind, and owner references (all CacheBackend-identity // dependent), so this returns only PodSpec.Containers and Service.Spec // Ports/Type. @@ -127,8 +121,7 @@ func ResolveRedisL2Server(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, * PeriodSeconds: 10, FailureThreshold: 6, }, - // Reuse the shared provider-resources helper plus the autoscaling CPU - // request fallback. The memory limit here is also what --maxmemory is + // Reuse the shared provider-resources helper. The memory limit here is also what --maxmemory is // derived from, so the two stay consistent. Resources: defaultServerResources(cache), } diff --git a/internal/adapters/builtin/storage/redis_test.go b/internal/adapters/builtin/storage/redis_test.go index 2eb96af6..7a82c747 100644 --- a/internal/adapters/builtin/storage/redis_test.go +++ b/internal/adapters/builtin/storage/redis_test.go @@ -142,8 +142,7 @@ func TestResolveRedisL2Server(t *testing.T) { func TestResolveRedisL2ServerResourceContract(t *testing.T) { // The renderer's resource contract, asserted on the surface its consumer uses // (the rendered container) rather than only on the shared helper: spec.remoteStorage.redis.resources - // is the operator-owned baseline and passes through; autoscaling adds the - // CPU-request fallback the HPA needs as a utilization denominator; and the + // is the operator-owned baseline and passes through; the // rendered resources must not ALIAS the CR — a caller mutating the pod it got // back would otherwise be writing into the CacheBackend's spec. t.Run("spec.remoteStorage.redis.resources passes through", func(t *testing.T) { @@ -161,19 +160,6 @@ func TestResolveRedisL2ServerResourceContract(t *testing.T) { } }) - t.Run("autoscaling adds the CPU-request fallback", func(t *testing.T) { - cb := withMemory(newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang"), "3Gi", "1Gi") - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - pod, _, err := ResolveRedisL2Server(cb) - if err != nil { - t.Fatalf("ResolveRedisL2Server: %v", err) - } - cpu := pod.Containers[0].Resources.Requests[corev1.ResourceCPU] - if cpu.IsZero() { - t.Fatalf("requests.cpu is zero/absent under autoscaling — a targetCPUUtilization HPA would divide by zero: %+v", pod.Containers[0].Resources) - } - }) - t.Run("rendered resources do not alias the CR", func(t *testing.T) { cb := withMemory(newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang"), "3Gi", "1Gi") pod, _, err := ResolveRedisL2Server(cb) @@ -345,8 +331,7 @@ func TestResolveRedisL2ServerNilCache(t *testing.T) { // TestResolveRedisL2ServerStaysClusterIP bounds the blast radius: the managed L2 // stays a plain in-cluster virtual IP — no hostNetwork, no NodePort — so it fits -// the engines-anywhere model (the whole reason Redis is the default L2, not the -// Mooncake mesh). +// the engines-anywhere model. func TestResolveRedisL2ServerStaysClusterIP(t *testing.T) { pod, svc, err := ResolveRedisL2Server(newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang")) if err != nil { diff --git a/internal/adapters/builtin/storage/registry.go b/internal/adapters/builtin/storage/registry.go index a0d42c4f..439fdace 100644 --- a/internal/adapters/builtin/storage/registry.go +++ b/internal/adapters/builtin/storage/registry.go @@ -34,19 +34,6 @@ type externalProvider struct { protocol backendadapter.Protocol } -type options struct { - lmCacheServerImage string -} - -// Option configures the shipping remote-storage provider adapters. -type Option func(*options) - -// WithLMCacheServerImage sets the operator-selected fallback image for managed -// LMCache servers. A per-CacheBackend image still takes precedence. -func WithLMCacheServerImage(image string) Option { - return func(opts *options) { opts.lmCacheServerImage = image } -} - func (p externalProvider) Supports(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) bool { return storage != nil && storage.Provider == p.provider && @@ -62,12 +49,7 @@ func (p externalProvider) Render(cache *cachev1alpha1.CacheBackend) (*backendada // DefaultRegistry returns the shipping provider capabilities. Engine/runtime // compatibility is intentionally not encoded here. -func DefaultRegistry(opts ...Option) *backendadapter.Registry { - var cfg options - for _, opt := range opts { - opt(&cfg) - } - +func DefaultRegistry() *backendadapter.Registry { registry := backendadapter.NewRegistry() registry.Register(managedProvider{ provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, @@ -76,27 +58,7 @@ func DefaultRegistry(opts ...Option) *backendadapter.Registry { return rendered(pod, service, backendadapter.ProtocolRESP, err) }, }) - registry.Register(managedProvider{ - provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - render: func(cache *cachev1alpha1.CacheBackend) (*backendadapter.RenderedStorage, error) { - pod, service, err := ResolveLMCacheServer(cache, cfg.lmCacheServerImage) - return rendered(pod, service, backendadapter.ProtocolLMCache, err) - }, - }) - registry.Register(managedProvider{ - provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - render: func(cache *cachev1alpha1.CacheBackend) (*backendadapter.RenderedStorage, error) { - pod, service, err := ResolveMooncakeServer(cache) - return rendered(pod, service, backendadapter.ProtocolMooncakeStore, err) - }, - }) - for _, provider := range []externalProvider{ - {provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, protocol: backendadapter.ProtocolRESP}, - {provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, protocol: backendadapter.ProtocolLMCache}, - {provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, protocol: backendadapter.ProtocolMooncakeStore}, - } { - registry.Register(provider) - } + registry.Register(externalProvider{provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, protocol: backendadapter.ProtocolRESP}) return registry } diff --git a/internal/adapters/builtin/storage/registry_test.go b/internal/adapters/builtin/storage/registry_test.go index ca138cf7..76891203 100644 --- a/internal/adapters/builtin/storage/registry_test.go +++ b/internal/adapters/builtin/storage/registry_test.go @@ -5,167 +5,36 @@ package storage import ( - "strings" + "errors" "testing" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" ) -const testLMCacheServerImage = "registry.example/lmcache:controller-default" - -func TestManagedRedisProviderOwnsTypedWorkloadConfig(t *testing.T) { - memory := resource.MustParse("2Gi") - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{ - Image: "registry.example/redis:test", - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{corev1.ResourceMemory: memory}, - }, - }, - }, - }, - } - - provider, err := DefaultRegistry().Select(cache.Spec.RemoteStorage) - if err != nil { - t.Fatalf("Select: %v", err) - } - rendered, err := provider.Render(cache) - if err != nil { - t.Fatalf("Render: %v", err) - } - if rendered.Protocol != "resp" { - t.Fatalf("protocol = %q, want resp", rendered.Protocol) - } - container := rendered.PodSpec.Containers[0] - if container.Image != "registry.example/redis:test" { - t.Fatalf("image = %q, want typed Redis image", container.Image) - } - if got := container.Resources.Limits[corev1.ResourceMemory]; got.Cmp(memory) != 0 { - t.Fatalf("memory limit = %s, want %s", got.String(), memory.String()) - } -} - -func TestCanonicalProviderUsesBoundedDefaults(t *testing.T) { - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, - }, - }, - } - - selected, err := DefaultRegistry().Select(cache.Spec.RemoteStorage) - if err != nil { - t.Fatalf("Select: %v", err) - } - rendered, err := selected.Render(cache) - if err != nil { - t.Fatalf("Render: %v", err) - } - container := rendered.PodSpec.Containers[0] - wantMemory := resource.MustParse("8Gi") - if got := container.Resources.Limits[corev1.ResourceMemory]; got.Cmp(wantMemory) != 0 { - t.Fatalf("canonical default memory limit = %s, want %s", got.String(), wantMemory.String()) - } -} - -func TestProviderRetainsBoundedResourcesWithoutDefaulter(t *testing.T) { - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - }, - } - rendered, _, err := ResolveLMCacheServer(cache, testLMCacheServerImage) - if err != nil { - t.Fatalf("ResolveLMCacheServer: %v", err) - } - wantLimit := resource.MustParse("8Gi") - wantRequest := resource.MustParse("4Gi") - resources := rendered.Containers[0].Resources - if got := resources.Limits[corev1.ResourceMemory]; got.Cmp(wantLimit) != 0 { - t.Fatalf("fallback memory limit = %s, want %s", got.String(), wantLimit.String()) - } - if got := resources.Requests[corev1.ResourceMemory]; got.Cmp(wantRequest) != 0 { - t.Fatalf("fallback memory request = %s, want %s", got.String(), wantRequest.String()) - } -} - -func TestLMCacheServerImageResolution(t *testing.T) { - t.Parallel() - - newCache := func(image string) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{ - Image: image, - }, - }, - }, +func TestDefaultRegistrySupportsRedisOwnershipModes(t *testing.T) { + registry := DefaultRegistry() + for _, ownership := range []cachev1alpha1.CacheBackendRemoteStorageOwnership{ + cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + } { + storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: ownership, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + if _, err := registry.Select(storage); err != nil { + t.Fatalf("Select(Redis, %s): %v", ownership, err) } } +} - for _, tc := range []struct { - name string - cacheImage string - controllerImage string - wantImage string - wantErr string - }{ - { - name: "controller fallback", - controllerImage: testLMCacheServerImage, - wantImage: testLMCacheServerImage, - }, - { - name: "CacheBackend override", - cacheImage: "registry.example/lmcache:per-cache", - controllerImage: testLMCacheServerImage, - wantImage: "registry.example/lmcache:per-cache", - }, - { - name: "missing image", - wantErr: "set spec.remoteStorage.lmCacheServer.image or controller --lmcache-server-image", - }, - } { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - pod, _, err := ResolveLMCacheServer(newCache(tc.cacheImage), tc.controllerImage) - if tc.wantErr != "" { - if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("ResolveLMCacheServer() error = %v, want containing %q", err, tc.wantErr) - } - return - } - if err != nil { - t.Fatalf("ResolveLMCacheServer(): %v", err) - } - if got := pod.Containers[0].Image; got != tc.wantImage { - t.Fatalf("container image = %q, want %q", got, tc.wantImage) - } - }) +func TestDefaultRegistryRejectsUnknownProvider(t *testing.T) { + _, err := DefaultRegistry().Select(&cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: "removed", + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + }) + if !errors.Is(err, backendadapter.ErrNoProvider) { + t.Fatalf("Select() error = %v, want ErrNoProvider", err) } } diff --git a/internal/cli/doctor/checks/cachebackend.go b/internal/cli/doctor/checks/cachebackend.go index e51b7ca5..dc4026c0 100644 --- a/internal/cli/doctor/checks/cachebackend.go +++ b/internal/cli/doctor/checks/cachebackend.go @@ -187,21 +187,25 @@ func matchedEnginePodCount(ctx context.Context, c client.Client, cb *cachev1alph // unreachable) for a backend, returning ok=false when the endpoint is healthy // (or when no dialer is configured to prove unreachability). func endpointFinding(ctx context.Context, cb *cachev1alpha1.CacheBackend, ref string, dial TCPDialer) (doctor.Finding, bool) { - if cb.Status.Endpoint == "" { + endpoint := "" + if cb.Status.RemoteStorage != nil { + endpoint = cb.Status.RemoteStorage.Endpoint + } + if endpoint == "" { return doctor.Finding{ Code: doctor.CodeBackendEndpointUnreachable, Status: doctor.StatusWarn, Check: checkCacheBackendHealth, Resource: ref, - Message: "status.endpoint is empty — clients have no address to reach this backend yet", + Message: "status.remoteStorage.endpoint is empty — the remote tier has no published address", }, true } if dial == nil { return doctor.Finding{}, false } - if err := dial(ctx, cb.Status.Endpoint); err != nil { + if err := dial(ctx, endpoint); err != nil { return doctor.Finding{ Code: doctor.CodeBackendEndpointUnreachable, Status: doctor.StatusWarn, Check: checkCacheBackendHealth, Resource: ref, - Message: fmt.Sprintf("status.endpoint %q is not reachable over TCP: %v", cb.Status.Endpoint, err), + Message: fmt.Sprintf("status.remoteStorage.endpoint %q is not reachable over TCP: %v", endpoint, err), }, true } return doctor.Finding{}, false diff --git a/internal/cli/doctor/checks/checks.go b/internal/cli/doctor/checks/checks.go index bd9eb7db..0d7c6d10 100644 --- a/internal/cli/doctor/checks/checks.go +++ b/internal/cli/doctor/checks/checks.go @@ -124,7 +124,7 @@ type Deps struct { // means doctor probes the unauthenticated path and flags the auth state. Token string - // DialTCP probes raw reachability of a CacheBackend's status.endpoint. nil + // DialTCP probes raw reachability of a CacheBackend's remote-storage endpoint. nil // disables the TCP sub-probe (endpoint presence is still checked). DialTCP TCPDialer diff --git a/internal/cli/doctor/checks/checks_test.go b/internal/cli/doctor/checks/checks_test.go index 1814e180..309cc51e 100644 --- a/internal/cli/doctor/checks/checks_test.go +++ b/internal/cli/doctor/checks/checks_test.go @@ -98,14 +98,14 @@ func healthyBackend(now time.Time) *cachev1alpha1.CacheBackend { Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "10.0.0.5:8200", + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageStatus{Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "10.0.0.5:8200", Ready: metav1.ConditionTrue}, MatchedEnginePods: ptr(int32(2)), Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "KVEventObserved", "ready")}, IndexParticipation: &cachev1alpha1.CacheBackendIndexParticipation{ @@ -404,9 +404,9 @@ func TestCacheBackendHealth(t *testing.T) { Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "missing"}}, }, @@ -532,14 +532,14 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: "h:1", }, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "h:1", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ok")}, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageStatus{Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "h:1", Ready: metav1.ConditionTrue}, + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ok")}, }, } c := fakeClient(t, cb) @@ -617,14 +617,14 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: "cache.example.com:8200", }, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "cache.example.com:8200", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageStatus{Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "cache.example.com:8200", Ready: metav1.ConditionTrue}, + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, }, } c := fakeClient(t, cb) @@ -641,14 +641,14 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: "cache.example.com:8200", }, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "cache.example.com:8200", - Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageStatus{Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "cache.example.com:8200", Ready: metav1.ConditionTrue}, + Conditions: []metav1.Condition{readyCond(metav1.ConditionTrue, "EndpointAccepted", "ready")}, }, } fs := CacheBackendHealth(ctx, fakeClient(t, cb), "", now, DefaultStaleWindow, okDial) @@ -665,7 +665,7 @@ func TestCacheBackendHealthMessageBranches(t *testing.T) { cb.Name = "host-only" cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.RemoteStorage = nil - cb.Status.Endpoint = "" + cb.Status.RemoteStorage = nil fs := CacheBackendHealth(ctx, fakeClient(t, cb), "", now, DefaultStaleWindow, okDial) if len(fs) != 1 || fs[0].Code != doctor.CodeBackendHealthy { t.Fatalf("canonical host-only backend should be CB006, got %v", codesOf(fs)) diff --git a/internal/cli/doctor/finding.go b/internal/cli/doctor/finding.go index c7dbafa6..5acc4ca3 100644 --- a/internal/cli/doctor/finding.go +++ b/internal/cli/doctor/finding.go @@ -152,7 +152,7 @@ const ( // CodeBackendStale: status.indexParticipation.lastEventAt is older than the // staleness window — KV events have stopped flowing. CodeBackendStale = "CB004" - // CodeBackendEndpointUnreachable: status.endpoint is empty or not reachable + // CodeBackendEndpointUnreachable: the remote-storage endpoint is empty or not reachable // over TCP. CodeBackendEndpointUnreachable = "CB005" // CodeBackendHealthy: the CacheBackend passed every per-backend check. diff --git a/internal/controller/cachebackend_autoscaling_test.go b/internal/controller/cachebackend_autoscaling_test.go deleted file mode 100644 index 06dd07cb..00000000 --- a/internal/controller/cachebackend_autoscaling_test.go +++ /dev/null @@ -1,516 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "testing" - - appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// ---- HPA reconciliation ----------------------------------------------------- - -func autoscalingBackend(name, namespace string, min, max int32, targetCPU *int32) *cachev1alpha1.CacheBackend { - cb := lmcacheBackend(name, namespace) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: ptrInt32(min), - MaxReplicas: max, - TargetCPUUtilizationPercent: targetCPU, - } - return cb -} - -func getHPA(t *testing.T, r *CacheBackendReconciler, name, namespace string) *autoscalingv2.HorizontalPodAutoscaler { - t.Helper() - var hpa autoscalingv2.HorizontalPodAutoscaler - if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &hpa); err != nil { - t.Fatalf("get HPA %s/%s: %v", namespace, name, err) - } - return &hpa -} - -func TestReconcileHPACreated(t *testing.T) { - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 2, 5, ptrInt32(60)) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - hpa := getHPA(t, r, "cache", "ns1") - if hpa.Spec.ScaleTargetRef.Kind != "Deployment" || hpa.Spec.ScaleTargetRef.Name != "cache" { - t.Fatalf("HPA target = %+v, want Deployment/cache", hpa.Spec.ScaleTargetRef) - } - if hpa.Spec.MinReplicas == nil || *hpa.Spec.MinReplicas != 2 || hpa.Spec.MaxReplicas != 5 { - t.Fatalf("HPA min/max = %v/%d, want 2/5", hpa.Spec.MinReplicas, hpa.Spec.MaxReplicas) - } - if owner := metav1.GetControllerOf(hpa); owner == nil || owner.Name != "cache" { - t.Fatalf("HPA controller owner = %+v, want CacheBackend/cache", owner) - } - if len(hpa.Spec.Metrics) != 1 { - t.Fatalf("HPA metrics = %d, want 1", len(hpa.Spec.Metrics)) - } - m := hpa.Spec.Metrics[0] - if m.Type != autoscalingv2.ResourceMetricSourceType || m.Resource == nil || m.Resource.Name != corev1.ResourceCPU { - t.Fatalf("HPA metric = %+v, want CPU resource metric", m) - } - if m.Resource.Target.AverageUtilization == nil || *m.Resource.Target.AverageUtilization != 60 { - t.Fatalf("HPA target CPU = %v, want 60", m.Resource.Target.AverageUtilization) - } -} - -func TestReconcileHPADefaults(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - hpa := getHPA(t, r, "cache", "ns1") - if hpa.Spec.MinReplicas == nil || *hpa.Spec.MinReplicas != defaultHPAMinReplicas { - t.Fatalf("default min replicas = %v, want %d", hpa.Spec.MinReplicas, defaultHPAMinReplicas) - } - target := hpa.Spec.Metrics[0].Resource.Target.AverageUtilization - if target == nil || *target != defaultHPATargetCPUUtilizationPercent { - t.Fatalf("default target CPU = %v, want %d", target, defaultHPATargetCPUUtilizationPercent) - } -} - -func TestReconcileHPAUpdated(t *testing.T) { - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 1, 3, ptrInt32(50)) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - live := getBackend(t, r, "cache", "ns1") - live.Spec.Autoscaling.MaxReplicas = 10 - live.Spec.Autoscaling.TargetCPUUtilizationPercent = ptrInt32(80) - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("update autoscaling: %v", err) - } - reconcile(t, r, "cache", "ns1") - - hpa := getHPA(t, r, "cache", "ns1") - if hpa.Spec.MaxReplicas != 10 { - t.Fatalf("HPA max after update = %d, want 10", hpa.Spec.MaxReplicas) - } - if got := hpa.Spec.Metrics[0].Resource.Target.AverageUtilization; got == nil || *got != 80 { - t.Fatalf("HPA target CPU after update = %v, want 80", got) - } -} - -func TestReconcileHPADeletedWhenAutoscalingCleared(t *testing.T) { - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 1, 3, nil) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - _ = getHPA(t, r, "cache", "ns1") - - live := getBackend(t, r, "cache", "ns1") - live.Spec.Autoscaling = nil - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("clear autoscaling: %v", err) - } - reconcile(t, r, "cache", "ns1") - - var hpas autoscalingv2.HorizontalPodAutoscalerList - if err := r.List(context.Background(), &hpas); err != nil { - t.Fatalf("list HPAs: %v", err) - } - if len(hpas.Items) != 0 { - t.Fatalf("HPAs = %d, want 0 after autoscaling cleared", len(hpas.Items)) - } -} - -func TestReconcileHPACleanedUpOnSwitchToExternal(t *testing.T) { - // Switching to an External backend sheds all managed children, including - // the HPA. - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 1, 3, nil) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - _ = getHPA(t, r, "cache", "ns1") - - live := getBackend(t, r, "cache", "ns1") - live.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - live.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - live.Spec.RemoteStorage = externalLMCacheStorage("external.ns1.svc:8080") - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("switch to external: %v", err) - } - reconcile(t, r, "cache", "ns1") - - var hpas autoscalingv2.HorizontalPodAutoscalerList - if err := r.List(context.Background(), &hpas); err != nil { - t.Fatalf("list HPAs: %v", err) - } - if len(hpas.Items) != 0 { - t.Fatalf("HPAs = %d, want 0 after switch to External", len(hpas.Items)) - } -} - -func TestReconcileInitialReplicasFromAutoscalingMin(t *testing.T) { - // With autoscaling configured, the Deployment must come up at the HPA's - // minReplicas — otherwise it briefly runs below the HPA floor on first - // apply (and may publish ScaledToZero status if spec.replicas defaults to - // zero on a different shape). - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 3, 6, nil) - // Even with spec.replicas explicitly set, the HPA's floor wins on init. - cb.Spec.Replicas = ptrInt32(1) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - dep := getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 3 { - t.Fatalf("initial deployment replicas = %v, want 3 (autoscaling.minReplicas)", dep.Spec.Replicas) - } -} - -func TestReconcileInitialReplicasDefaultsToOneWithAutoscaling(t *testing.T) { - // Autoscaling without minReplicas → default 1 (matching the HPA default - // the controller renders). - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 5} - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - dep := getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 1 { - t.Fatalf("initial deployment replicas = %v, want 1 (default autoscaling floor)", dep.Spec.Replicas) - } -} - -func TestReconcileDeploymentClampsToRaisedHPAFloor(t *testing.T) { - // When the user raises autoscaling.minReplicas above the current live - // replica count, the reconciler must NOT preserve the stale lower value — - // otherwise managedReadiness would report Ready against the old count - // before the HPA controller catches up, briefly publishing a Ready that - // does not satisfy the new minimum. - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 1, 5, nil) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - dep := getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 1 { - t.Fatalf("initial deployment replicas = %v, want 1", dep.Spec.Replicas) - } - - // Raise the HPA floor; live replicas (set by us above) lags behind. - live := getBackend(t, r, "cache", "ns1") - *live.Spec.Autoscaling.MinReplicas = 4 - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("raise minReplicas: %v", err) - } - reconcile(t, r, "cache", "ns1") - - dep = getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas < 4 { - t.Fatalf("deployment replicas = %v, want >= 4 (clamped to raised HPA floor)", dep.Spec.Replicas) - } -} - -func TestReconcileDeploymentRespectsHPAReplicas(t *testing.T) { - // When an HPA owns the replica count, the reconciler must not overwrite - // dep.Spec.Replicas back to spec.Replicas — that would let the controller - // and the HPA fight, churning the rollout. - scheme := newScheme(t) - cb := autoscalingBackend("cache", "ns1", 1, 5, nil) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - // HPA scales the Deployment to 4 replicas (simulated). - dep := getDeployment(t, r, "cache", "ns1") - scaled := int32(4) - dep.Spec.Replicas = &scaled - if err := r.Update(context.Background(), dep); err != nil { - t.Fatalf("update deployment replicas: %v", err) - } - reconcile(t, r, "cache", "ns1") - - dep = getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 4 { - t.Fatalf("deployment replicas = %v, want 4 (HPA-managed, not reset by reconciler)", dep.Spec.Replicas) - } -} - -// ---- Status (Progressing, observedGeneration) ------------------------------- - -func TestStatusProgressingTrueWhilePending(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - updated := getBackend(t, r, "cache", "ns1") - ready := findCondition(updated.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonRolloutInProgress { - t.Fatalf("Ready condition = %+v, want False/RolloutInProgress right after create", ready) - } - prog := findCondition(updated.Status.Conditions, conditionTypeProgressing) - if prog == nil || prog.Status != metav1.ConditionTrue { - t.Fatalf("Progressing condition = %+v, want True while Pending", prog) - } -} - -func TestStatusProgressingFalseOnceReady(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - dep := getDeployment(t, r, "cache", "ns1") - dep.Status.ObservedGeneration = dep.Generation - dep.Status.Replicas = 1 - dep.Status.UpdatedReplicas = 1 - dep.Status.AvailableReplicas = 1 - dep.Status.ReadyReplicas = 1 - if err := r.Status().Update(context.Background(), dep); err != nil { - t.Fatalf("update deployment status: %v", err) - } - reconcile(t, r, "cache", "ns1") - - updated := getBackend(t, r, "cache", "ns1") - ready := findCondition(updated.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionTrue { - t.Fatalf("Ready condition = %+v, want True", ready) - } - prog := findCondition(updated.Status.Conditions, conditionTypeProgressing) - if prog == nil || prog.Status != metav1.ConditionFalse || prog.Reason != "Synced" { - t.Fatalf("Progressing condition = %+v, want False/Synced once Ready", prog) - } -} - -func TestStatusProgressingFalseWhenDegraded(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - // Simulate a rolled-out Deployment that has lost some replicas: rollout - // has finished (Progressing should be False) but Ready is False because - // not enough replicas are available. - dep := getDeployment(t, r, "cache", "ns1") - dep.Status.ObservedGeneration = dep.Generation - dep.Status.Replicas = 2 - dep.Status.UpdatedReplicas = 2 - dep.Status.AvailableReplicas = 1 - dep.Status.ReadyReplicas = 1 - if err := r.Status().Update(context.Background(), dep); err != nil { - t.Fatalf("update deployment status: %v", err) - } - reconcile(t, r, "cache", "ns1") - - updated := getBackend(t, r, "cache", "ns1") - ready := findCondition(updated.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != conditionReasonReplicasUnavailable { - t.Fatalf("Ready condition = %+v, want False/ReplicasUnavailable", ready) - } - prog := findCondition(updated.Status.Conditions, conditionTypeProgressing) - if prog == nil || prog.Status != metav1.ConditionFalse || prog.Reason != "Degraded" { - t.Fatalf("Progressing condition = %+v, want False/Degraded", prog) - } -} - -func TestStatusProgressingFalseAtScaledToZero(t *testing.T) { - // A backend with spec.replicas: 0 is in a stable terminal state — no - // rollout is in motion. Progressing must be False (Reason=ScaledToZero), - // not True, so consumers don't see "still converging" forever. - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(0) - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - updated := getBackend(t, r, "cache", "ns1") - prog := findCondition(updated.Status.Conditions, conditionTypeProgressing) - if prog == nil || prog.Status != metav1.ConditionFalse || prog.Reason != "ScaledToZero" { - t.Fatalf("Progressing condition = %+v, want False/ScaledToZero at zero replicas", prog) - } -} - -func TestStatusObservedGenerationTracksSpec(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - if got := getBackend(t, r, "cache", "ns1").Status.ObservedGeneration; got != 1 { - t.Fatalf("initial observedGeneration = %d, want 1", got) - } - - // Bump the spec → bump generation. - live := getBackend(t, r, "cache", "ns1") - live.Generation = 5 - live.Spec.Replicas = ptrInt32(3) - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("update spec: %v", err) - } - reconcile(t, r, "cache", "ns1") - - if got := getBackend(t, r, "cache", "ns1").Status.ObservedGeneration; got != 5 { - t.Fatalf("status.observedGeneration after update = %d, want 5", got) - } -} - -// ---- Pure-function coverage ------------------------------------------------- - -func TestProgressingFromReadyExhaustive(t *testing.T) { - cases := []struct { - name string - readyStatus metav1.ConditionStatus - reason string - wantStatus metav1.ConditionStatus - wantReason string - }{ - {"Ready", metav1.ConditionTrue, conditionReasonBackendReady, metav1.ConditionFalse, "Synced"}, - {"Pending-rollout", metav1.ConditionFalse, conditionReasonRolloutInProgress, metav1.ConditionTrue, conditionReasonRolloutInProgress}, - {"Pending-scaled-to-zero", metav1.ConditionFalse, conditionReasonScaledToZero, metav1.ConditionFalse, conditionReasonScaledToZero}, - {"Degraded", metav1.ConditionFalse, conditionReasonReplicasUnavailable, metav1.ConditionFalse, "Degraded"}, - // The KV-event gate's AwaitingFirstKVEvent is still-converging; without - // this case the controller would advertise a non-degraded wait window - // as stuck (Progressing=False), contradicting the documented contract. - {"Awaiting-first-kv-event", metav1.ConditionFalse, reasonAwaitingFirstKVEvent, metav1.ConditionTrue, reasonAwaitingFirstKVEvent}, - // The gate's NoKVEventsObserved is a stable failure (Degraded); - // already not progressing — same shape as ReplicasUnavailable. - {"No-kv-events-observed", metav1.ConditionFalse, reasonNoKVEventsObserved, metav1.ConditionFalse, reasonNoKVEventsObserved}, - {"Unknown-reason-passthrough", metav1.ConditionFalse, "WedgedExternalEndpoint", metav1.ConditionFalse, "WedgedExternalEndpoint"}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - status, reason, _ := progressingFromReady(tc.readyStatus, tc.reason, "msg") - if status != tc.wantStatus { - t.Fatalf("status = %v, want %v", status, tc.wantStatus) - } - if reason != tc.wantReason { - t.Fatalf("reason = %q, want %q", reason, tc.wantReason) - } - }) - } -} - -func TestDesiredReplicasPrefersHPAWhenAutoscalingSet(t *testing.T) { - cb := autoscalingBackend("cache", "ns1", 1, 5, nil) - // User-set spec.replicas should be ignored once autoscaling is in charge — - // the HPA's writes to dep.spec.replicas are authoritative. - cb.Spec.Replicas = ptrInt32(1) - dep := newDep(4) - if got := desiredReplicas(cb, dep); got != 4 { - t.Fatalf("desiredReplicas = %d, want 4 (HPA-driven)", got) - } -} - -func TestDesiredReplicasFallbackToSpecWhenNoAutoscaling(t *testing.T) { - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(3) - dep := newDep(7) // out-of-band edit; not HPA-managed. - if got := desiredReplicas(cb, dep); got != 3 { - t.Fatalf("desiredReplicas = %d, want 3 (spec.replicas wins without autoscaling)", got) - } -} - -func TestDesiredReplicasReflectsSingletonClamp(t *testing.T) { - // A singleton cache-server is clamped to one replica at deploy time. desiredReplicas - // — the readiness expectation — must reflect the clamp, or a grandfathered - // spec.replicas:3 (written before admission rejected it) deploys one pod but - // expects three and reports RolloutInProgress forever. - t.Run("sglang Redis L2 (pair-driven) clamps to 1", func(t *testing.T) { - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, - } - cb.Spec.Replicas = ptrInt32(3) - if got := desiredReplicas(cb, newDep(3)); got != 1 { - t.Fatalf("desiredReplicas = %d, want 1 (singleton readiness must match the clamp)", got) - } - }) - t.Run("host-network master (hostNetwork-driven) clamps to 1", func(t *testing.T) { - cb := mooncakeBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(3) - dep := newDep(3) - dep.Spec.Template.Spec.HostNetwork = true - if got := desiredReplicas(cb, dep); got != 1 { - t.Fatalf("desiredReplicas = %d, want 1 (host-network singleton)", got) - } - }) - t.Run("disabled (0) is preserved, not clamped up", func(t *testing.T) { - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - cb.Spec.Replicas = ptrInt32(0) - if got := desiredReplicas(cb, newDep(0)); got != 0 { - t.Fatalf("desiredReplicas = %d, want 0 (disabled preserved)", got) - } - }) - t.Run("EventsOnly is NOT a singleton — no cache-server is rendered", func(t *testing.T) { - cb := lmcacheBackend("cache", "ns1") - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - } - cb.Spec.Replicas = ptrInt32(3) - if got := desiredReplicas(cb, newDep(3)); got != 3 { - t.Fatalf("desiredReplicas = %d, want 3 (EventsOnly provisions no Redis, so nothing to clamp)", got) - } - }) - t.Run("vllm+LMCache is NOT a singleton — spec.replicas honored", func(t *testing.T) { - cb := lmcacheBackend("cache", "ns1") // engine defaults to vllm - cb.Spec.Replicas = ptrInt32(3) - if got := desiredReplicas(cb, newDep(3)); got != 3 { - t.Fatalf("desiredReplicas = %d, want 3 (vLLM lm:// server scales, not a singleton)", got) - } - }) -} - -func TestManagedReadinessIgnoresSpecReplicasUnderHPA(t *testing.T) { - // spec.replicas=0 with autoscaling set must NOT trip the ScaledToZero - // guard — the HPA owns the count, and minReplicas>=1 is enforced by the - // kubebuilder validation on autoscaling.minReplicas. - cb := autoscalingBackend("cache", "ns1", 1, 3, nil) - cb.Spec.Replicas = ptrInt32(0) - dep := newDep(2) - dep.Status.ObservedGeneration = dep.Generation - dep.Status.UpdatedReplicas = 2 - dep.Status.AvailableReplicas = 2 - - status, reason, _ := managedReadiness(cb, dep) - if status != metav1.ConditionTrue || reason != conditionReasonBackendReady { - t.Fatalf("managedReadiness = %v/%q, want True/BackendReady under HPA with 2/2 replicas", status, reason) - } -} - -func newDep(replicas int32) *appsv1.Deployment { - r := replicas - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Generation: 1}, - Spec: appsv1.DeploymentSpec{Replicas: &r}, - } -} diff --git a/internal/controller/cachebackend_dispatch.go b/internal/controller/cachebackend_dispatch.go index afdb9a90..1f7a2a1e 100644 --- a/internal/controller/cachebackend_dispatch.go +++ b/internal/controller/cachebackend_dispatch.go @@ -17,8 +17,8 @@ import ( // dispatch routes a CacheBackend by integration mode and effective remote // storage ownership. EventsOnly and canonical host-only configurations shed // managed provider workloads; External storage mirrors its configured endpoint -// to status; Managed Redis, LMCacheServer, and Mooncake storage is rendered by -// the selected runtime/provider adapter. Unsupported combinations also shed any +// to status; Managed Redis storage is rendered by the selected provider +// adapter. Unsupported combinations also shed any // previously managed workload. func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend) (ctrl.Result, error) { if r.Registry == nil || r.BackendRegistry == nil { @@ -34,7 +34,7 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge // generation owned, then run the server-less status path (the KV-event // readiness gate, no Service/endpoint/cascade). Checked before the // StatefulSet routing because the mode decides provisioning regardless of - // deploymentKind (a server-less backend ignores deploymentKind). + // provider workload. // // EventsOnly is checked before external remote-storage ownership so it takes // precedence over provider lifecycle. An admission-bypassed object carrying @@ -105,16 +105,6 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge return ctrl.Result{}, r.reconcileExternal(ctx, backend) } - // StatefulSet (per-replica PVCs via volumeClaimTemplates) is a later - // module. Phase 1 manages a Deployment only. SGLangHiCache is engine-local, - // so the schema-defaulted deploymentKind is inert for it. - if backend.Spec.DeploymentKind == cachev1alpha1.CacheBackendDeploymentKindStatefulSet && - storage != nil { - logger.V(1).Info("StatefulSet deploymentKind not yet supported; skipping", - "namespace", backend.Namespace, "name", backend.Name) - return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) - } - if storage == nil { if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { return ctrl.Result{}, err diff --git a/internal/controller/cachebackend_events_only_integration_test.go b/internal/controller/cachebackend_events_only_integration_test.go index 7784ff5d..65ddedb2 100644 --- a/internal/controller/cachebackend_events_only_integration_test.go +++ b/internal/controller/cachebackend_events_only_integration_test.go @@ -94,8 +94,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { } cb := getBackend(t, r, "cache", ns) - if cb.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want empty (no provisioned server)", cb.Status.Endpoint) + if cb.Status.RemoteStorage != nil { + t.Fatalf("status.remoteStorage = %+v, want nil (no configured remote tier)", cb.Status.RemoteStorage) } // Before any KV event, with the default 5m firstEventTimeout still @@ -136,8 +136,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { if deps, svcs := countOwnedWorkloads(t, k8s, "cache", ns); deps != 0 || svcs != 0 { t.Fatalf("post-event owned workloads = %d/%d, want 0/0", deps, svcs) } - if got.Status.Endpoint != "" { - t.Fatalf("post-event status.endpoint = %q, want empty", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("post-event status.remoteStorage = %+v, want nil", got.Status.RemoteStorage) } if c := findCondition(got.Status.Conditions, conditionTypeFunctionalProbeOK); c != nil { t.Fatalf("post-event FunctionalProbeOK = %+v, want absent", c) @@ -189,8 +189,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { if deps, svcs := countOwnedWorkloads(t, k8s, "cache", ns); deps != 0 || svcs != 0 { t.Fatalf("post-timeout owned workloads = %d/%d, want 0/0 (events-only provisions nothing)", deps, svcs) } - if got.Status.Endpoint != "" { - t.Fatalf("post-timeout status.endpoint = %q, want empty", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("post-timeout status.remoteStorage = %+v, want nil", got.Status.RemoteStorage) } }) @@ -200,21 +200,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { // reconcile (cleanupOwnedWorkload), end with empty status.endpoint, and // drop the managed-only advisory conditions. ns := freshNS(t, k8s) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: ns, Generation: 1}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, - }, - }, - } + cb := lmcacheBackend("cache", ns) + delete(cb.Annotations, annotationRequireKVEvents) if err := k8s.Create(ctx, cb); err != nil { t.Fatalf("create Offload backend: %v", err) } @@ -266,6 +253,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { // LMCache, no autoscaling.) live := getBackend(t, r, "cache", ns) live.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly + live.Spec.RemoteStorage = nil + live.Spec.LMCache = nil if err := k8s.Update(ctx, live); err != nil { t.Fatalf("update to EventsOnly: %v", err) } @@ -281,8 +270,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { } got := getBackend(t, r, "cache", ns) - if got.Status.Endpoint != "" { - t.Fatalf("post-flip status.endpoint = %q, want empty", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("post-flip status.remoteStorage = %+v, want nil", got.Status.RemoteStorage) } if c := findCondition(got.Status.Conditions, conditionTypeFunctionalProbeOK); c != nil { t.Fatalf("post-flip FunctionalProbeOK = %+v, want absent", c) @@ -322,7 +311,11 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { live := getBackend(t, r, "cache", ns) beforeSeed := live.DeepCopy() stale := metav1.NewTime(time.Now().Add(-time.Hour)) - live.Status.Endpoint = "cache." + ns + ".svc.cluster.local:8080" + live.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Endpoint: "cache." + ns + ".svc.cluster.local:6379", + Ready: metav1.ConditionTrue, + } live.Status.FirstAvailableAt = &stale meta.SetStatusCondition(&live.Status.Conditions, metav1.Condition{ Type: conditionTypeReady, Status: metav1.ConditionFalse, @@ -360,8 +353,8 @@ func TestIntegrationEventsOnlyMode(t *testing.T) { if got.Status.FirstAvailableAt == nil || got.Status.FirstAvailableAt.Time.Before(time.Now().Add(-time.Minute)) { t.Fatalf("post-flip firstAvailableAt = %v, want re-anchored to ~now (stale Offload anchor reused)", got.Status.FirstAvailableAt) } - if got.Status.Endpoint != "" { - t.Fatalf("post-flip status.endpoint = %q, want empty", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("post-flip status.remoteStorage = %+v, want nil", got.Status.RemoteStorage) } }) } diff --git a/internal/controller/cachebackend_events_test.go b/internal/controller/cachebackend_events_test.go index 1b5257f0..790b7e1a 100644 --- a/internal/controller/cachebackend_events_test.go +++ b/internal/controller/cachebackend_events_test.go @@ -7,12 +7,15 @@ package controller import ( "context" "errors" + "fmt" + "strconv" "strings" "sync/atomic" "testing" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -26,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/interceptor" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" ) // newReconcilerWithRecorder builds a reconciler wired with a buffered fake @@ -33,6 +37,10 @@ import ( // emit; assertions read from rec.Events with a select+default to avoid hanging // when an expected event is missing. func newReconcilerWithRecorder(t *testing.T, objs ...client.Object) (*CacheBackendReconciler, *events.FakeRecorder) { + return newReconcilerWithRecorderOptions(t, false, objs...) +} + +func newReconcilerWithRecorderOptions(t *testing.T, addReadyEngine bool, objs ...client.Object) (*CacheBackendReconciler, *events.FakeRecorder) { t.Helper() scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { @@ -41,6 +49,15 @@ func newReconcilerWithRecorder(t *testing.T, objs ...client.Object) (*CacheBacke if err := cachev1alpha1.AddToScheme(scheme); err != nil { t.Fatalf("add cache scheme: %v", err) } + if addReadyEngine { + for _, obj := range objs { + cb, ok := obj.(*cachev1alpha1.CacheBackend) + if !ok || !isTypedLMCachePodLocal(cb) { + continue + } + objs = append(objs, readyEnginePodForBackend(cb)) + } + } c := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). @@ -52,6 +69,41 @@ func newReconcilerWithRecorder(t *testing.T, objs ...client.Object) (*CacheBacke return r, rec } +func readyEnginePodForBackend(cb *cachev1alpha1.CacheBackend) *corev1.Pod { + if cb.UID == "" { + cb.UID = types.UID(fmt.Sprintf("uid-%s", cb.Name)) + } + if cb.Spec.EngineSelector == nil || len(cb.Spec.EngineSelector.MatchLabels) == 0 { + cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": cb.Name + "-engine"}} + } + labels := make(map[string]string, len(cb.Spec.EngineSelector.MatchLabels)) + for key, value := range cb.Spec.EngineSelector.MatchLabels { + labels[key] = value + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: cb.Name + "-engine", Namespace: cb.Namespace, Labels: labels, + Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: cb.Namespace + "/" + cb.Name, + enginebinding.AnnotationInjectedByUID: string(cb.UID), + enginebinding.AnnotationInjectedGeneration: strconv.FormatInt(cb.Generation, 10), + }, + }, + Status: corev1.PodStatus{ + InitContainerStatuses: []corev1.ContainerStatus{{Name: lmCacheMPServerStatusContainerName, Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + }, + } +} + +func requireRemoteStorageForReadiness(cb *cachev1alpha1.CacheBackend) { + failOpen := false + if cb.Spec.Integration == nil { + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} + } + cb.Spec.Integration.FailOpen = &failOpen +} + // drainEvents pulls every event currently on the recorder channel. The channel // is non-blocking; absence of an expected event is detected by length, not by // blocking, so tests fail fast instead of timing out. @@ -134,39 +186,35 @@ func reconcileN(t *testing.T, r *CacheBackendReconciler, name, namespace string, func TestReconcileEmitsBackendDegradedOnTransition(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) - r, rec := newReconcilerWithRecorder(t, cb) + requireRemoteStorageForReadiness(cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) // Cold start: Pending → no event yet. reconcile(t, r, "cache", "ns1") - if events := drainEvents(rec); len(events) != 0 { - t.Fatalf("unexpected events on cold start: %v", events) - } + expectNoEvent(t, drainEvents(rec), eventReasonBackendDegraded) // Drive to Ready: no event for Pending → Ready (only Degraded entry/exit // is loud enough to deserve an event by design). - markDeploymentReady(t, r, "cache", "ns1", 2) + markDeploymentReady(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") if !isReady(getBackend(t, r, "cache", "ns1")) { t.Fatalf("Ready condition not True before degrading") } - if events := drainEvents(rec); len(events) != 0 { - t.Fatalf("unexpected events on Ready transition: %v", events) - } + expectNoEvent(t, drainEvents(rec), eventReasonBackendDegraded) // Backend dies under load: AvailableReplicas drops to 0 with the rollout // already observed → managedReadiness reports Ready=False/ReplicasUnavailable. - markDeploymentDegraded(t, r, "cache", "ns1", 2) + markDeploymentDegraded(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") updated := getBackend(t, r, "cache", "ns1") cond := findCondition(updated.Status.Conditions, conditionTypeReady) - if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonReplicasUnavailable { - t.Fatalf("Ready condition = %+v, want False/ReplicasUnavailable", cond) + if cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != reasonRemoteStorageUnavailable { + t.Fatalf("Ready condition = %+v, want False/RemoteStorageUnavailable", cond) } events := drainEvents(rec) - expectEvent(t, events, "Warning "+eventReasonBackendDegraded) - expectEvent(t, events, "0/2 replicas available") + expectEvent(t, events, "Warning "+reasonRemoteStorageUnavailable) + expectEvent(t, events, "0/1 replicas available") } // TestReconcileEmitsTransitionEventEvenWhenApplyErrors guards the @@ -186,7 +234,8 @@ func TestReconcileEmitsTransitionEventEvenWhenApplyErrors(t *testing.T) { } cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) + requireRemoteStorageForReadiness(cb) + enginePod := readyEnginePodForBackend(cb) // Block Deployment Updates after the first reconcile so the second pass // returns an apply error while the live Deployment status drives a @@ -207,7 +256,7 @@ func TestReconcileEmitsTransitionEventEvenWhenApplyErrors(t *testing.T) { c := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). - WithObjects(cb). + WithObjects(cb, enginePod). WithInterceptorFuncs(funcs). Build() rec := events.NewFakeRecorder(16) @@ -217,7 +266,7 @@ func TestReconcileEmitsTransitionEventEvenWhenApplyErrors(t *testing.T) { // First pass establishes the Deployment + drives Ready (no events; only // Degraded transitions are loud by design). reconcile(t, r, "cache", "ns1") - markDeploymentReady(t, r, "cache", "ns1", 2) + markDeploymentReady(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") _ = drainEvents(rec) @@ -227,12 +276,17 @@ func TestReconcileEmitsTransitionEventEvenWhenApplyErrors(t *testing.T) { // drives the readiness transition. blockUpdate.Store(true) live := getBackend(t, r, "cache", "ns1") - live.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v9" + live.Spec.RemoteStorage.Redis.Image = "example.com/redis:v9" live.Generation = 2 if err := r.Update(context.Background(), live); err != nil { t.Fatalf("update CR: %v", err) } - markDeploymentDegraded(t, r, "cache", "ns1", 2) + enginePod.Annotations[enginebinding.AnnotationInjectedGeneration] = "2" + if err := r.Update(context.Background(), enginePod); err != nil { + t.Fatalf("update engine Pod: %v", err) + } + r.refreshLMCacheMPConnectorStatus(context.Background(), getBackend(t, r, "cache", "ns1")) + markDeploymentDegraded(t, r, "cache", "ns1", 1) if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: "cache", Namespace: "ns1"}, @@ -240,11 +294,11 @@ func TestReconcileEmitsTransitionEventEvenWhenApplyErrors(t *testing.T) { t.Fatalf("reconcile returned nil, want error (apply was blocked)") } - if !isDegraded(getBackend(t, r, "cache", "ns1")) { - t.Fatalf("Ready condition not False/ReplicasUnavailable (status path runs independently of apply error)") + if ready := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady); ready == nil || ready.Reason != reasonRemoteStorageUnavailable { + t.Fatalf("Ready condition = %+v, want RemoteStorageUnavailable (status path runs independently of apply error)", ready) } events := drainEvents(rec) - expectEvent(t, events, "Warning "+eventReasonBackendDegraded) + expectEvent(t, events, "Warning "+reasonRemoteStorageUnavailable) } // TestReconcileNoPhantomEventOnStatusPatchFailure pins the rollback semantics @@ -264,7 +318,8 @@ func TestReconcileNoPhantomEventOnStatusPatchFailure(t *testing.T) { } cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) + requireRemoteStorageForReadiness(cb) + enginePod := readyEnginePodForBackend(cb) var blockStatusPatch atomic.Bool funcs := interceptor.Funcs{ @@ -280,7 +335,7 @@ func TestReconcileNoPhantomEventOnStatusPatchFailure(t *testing.T) { c := fake.NewClientBuilder(). WithScheme(scheme). WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). - WithObjects(cb). + WithObjects(cb, enginePod). WithInterceptorFuncs(funcs). Build() rec := events.NewFakeRecorder(16) @@ -290,7 +345,7 @@ func TestReconcileNoPhantomEventOnStatusPatchFailure(t *testing.T) { // Drive to Ready first. Pending → Ready emits no event by design (only // Degraded entry/exit are loud). reconcile(t, r, "cache", "ns1") - markDeploymentReady(t, r, "cache", "ns1", 2) + markDeploymentReady(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") _ = drainEvents(rec) @@ -299,7 +354,7 @@ func TestReconcileNoPhantomEventOnStatusPatchFailure(t *testing.T) { // error — and must emit NO event, because the apiserver never saw the // transition. blockStatusPatch.Store(true) - markDeploymentDegraded(t, r, "cache", "ns1", 2) + markDeploymentDegraded(t, r, "cache", "ns1", 1) if _, err := r.Reconcile(context.Background(), ctrl.Request{ NamespacedName: types.NamespacedName{Name: "cache", Namespace: "ns1"}, @@ -324,26 +379,26 @@ func TestReconcileNoPhantomEventOnStatusPatchFailure(t *testing.T) { }); err != nil { t.Fatalf("reconcile after unblock: %v", err) } - if !isDegraded(getBackend(t, r, "cache", "ns1")) { - t.Fatalf("Ready condition not False/ReplicasUnavailable after unblock") + if ready := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady); ready == nil || ready.Reason != reasonRemoteStorageUnavailable { + t.Fatalf("Ready condition = %+v, want RemoteStorageUnavailable after unblock", ready) } got := drainEvents(rec) - expectEvent(t, got, "Warning "+eventReasonBackendDegraded) + expectEvent(t, got, "Warning "+reasonRemoteStorageUnavailable) count := 0 for _, e := range got { - if strings.Contains(e, eventReasonBackendDegraded) { + if strings.Contains(e, reasonRemoteStorageUnavailable) { count++ } } if count != 1 { - t.Fatalf("BackendDegraded event count = %d, want exactly 1: %v", count, got) + t.Fatalf("RemoteStorageUnavailable event count = %d, want exactly 1: %v", count, got) } } func TestReconcileEmitsBackendRecoveredOnReadyTransition(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) - r, rec := newReconcilerWithRecorder(t, cb) + requireRemoteStorageForReadiness(cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") markDeploymentReady(t, r, "cache", "ns1", 1) @@ -362,18 +417,20 @@ func TestReconcileEmitsBackendRecoveredOnReadyTransition(t *testing.T) { t.Fatalf("Ready condition not True after recovery") } events := drainEvents(rec) - expectEvent(t, events, "Normal "+eventReasonBackendRecovered) + expectEvent(t, events, "Normal "+reasonRemoteStorageReady) // No spurious second warning during recovery. expectNoEvent(t, events, eventReasonBackendDegraded) } func TestReconcileSteadyStateDoesNotFloodEvents(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) - r, rec := newReconcilerWithRecorder(t, cb) + requireRemoteStorageForReadiness(cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") markDeploymentReady(t, r, "cache", "ns1", 1) + reconcile(t, r, "cache", "ns1") + _ = drainEvents(rec) // Five steady-state reconciles after Ready is established must not emit // any events — Ready→Ready is the no-op transition that mattered most for @@ -388,9 +445,8 @@ func TestReconcileSteadyStateDoesNotFloodEvents(t *testing.T) { func TestReconcileEmitsFailClosedWarningOnApply(t *testing.T) { failOpen := false cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: &failOpen} - r, rec := newReconcilerWithRecorder(t, cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) // First reconcile: previous status.failOpen is nil (effective true), spec // is false → transition fires the FailClosedEnabled Warning. The status @@ -417,9 +473,8 @@ func TestReconcileEmitsFailClosedWarningOnApply(t *testing.T) { func TestReconcileEmitsFailOpenRestoredWhenFlippedBack(t *testing.T) { failOpen := false cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: &failOpen} - r, rec := newReconcilerWithRecorder(t, cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") _ = drainEvents(rec) // discard the FailClosedEnabled warning emitted above @@ -443,7 +498,6 @@ func TestReconcileEmitsFailOpenRestoredWhenFlippedBack(t *testing.T) { func TestReconcileDefaultFailOpenIsSilent(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) r, rec := newReconcilerWithRecorder(t, cb) reconcile(t, r, "cache", "ns1") @@ -464,7 +518,6 @@ func TestReconcileNilRecorderIsSafe(t *testing.T) { // directly in tests and may be in tests that don't care about events. scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) r := newReconciler(scheme, cb) // no Recorder reconcile(t, r, "cache", "ns1") markDeploymentDegraded(t, r, "cache", "ns1", 1) diff --git a/internal/controller/cachebackend_hostnetwork_test.go b/internal/controller/cachebackend_hostnetwork_test.go deleted file mode 100644 index db8b4eaa..00000000 --- a/internal/controller/cachebackend_hostnetwork_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "testing" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// TestBuildDeploymentRecreateStrategyForHostNetwork covers the rollout strategy a -// hostNetwork cache-server needs. Such a pod binds its ports directly on the node, -// so the default RollingUpdate would surge a second pod onto the same host ports: -// it CrashLoops failing to bind while the old pod still holds them (in practice the -// scheduler rejects it earlier still, since the apiserver defaults -// hostPort=containerPort for hostNetwork pods). Recreate tears the old pod down -// first. Backends that stay on the pod network must keep the default strategy. -func TestBuildDeploymentRecreateStrategyForHostNetwork(t *testing.T) { - r := &CacheBackendReconciler{} - const ns = "default" - - t.Run("HostNetworkGetsRecreate", func(t *testing.T) { - dep := r.buildDeployment(mooncakeBackend("cache", ns), &corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{{Name: "master"}}, - }) - if got := dep.Spec.Strategy.Type; got != appsv1.RecreateDeploymentStrategyType { - t.Fatalf("strategy = %q, want %q (hostNetwork pods collide on node ports)", - got, appsv1.RecreateDeploymentStrategyType) - } - if !dep.Spec.Template.Spec.HostNetwork { - t.Fatal("hostNetwork was not propagated into the pod template") - } - }) - - t.Run("PodNetworkKeepsDefaultStrategy", func(t *testing.T) { - dep := r.buildDeployment(lmcacheBackend("cache", ns), &corev1.PodSpec{ - Containers: []corev1.Container{{Name: "server"}}, - }) - if got := dep.Spec.Strategy.Type; got != "" { - t.Fatalf("strategy = %q, want empty (apiserver defaults to RollingUpdate)", got) - } - }) -} - -// TestClampSingletonReplicas pins the reconciler's last line of defense. -// Admission rejects spec.replicas>1 / spec.autoscaling for a singleton backend, but -// ValidateUpdate only rejects violations an edit *introduces* — an object written -// before the rule existed stays in etcd with replicas=3 and is never re-validated. -// Rendering that faithfully would put several servers on the cluster: host-network -// masters contending for node ports or splitting the store, or several Redis pods -// partitioning the (sglang, LMCache) L2 keyspace. So the reconciler clamps rather -// than obeys — for BOTH singleton reasons. -func TestClampSingletonReplicas(t *testing.T) { - i32 := func(v int32) *int32 { return &v } - // sglangLMCache / vllmLMCache: the pair drives singleton-ness on the pod network - // (no hostNetwork), so these isolate the (sglang, LMCache) trigger from the - // host-network one. - sglangLMCache := func() *cachev1alpha1.CacheBackend { - cb := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }} - return cb - } - vllmLMCache := func() *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }} - } - for _, tc := range []struct { - name string - backend *cachev1alpha1.CacheBackend - hostNetwork bool - replicas *int32 - want *int32 - }{ - {"HostNetworkGrandfatheredScaleOutClamped", vllmLMCache(), true, i32(3), i32(1)}, - {"HostNetworkSingletonUntouched", vllmLMCache(), true, i32(1), i32(1)}, - {"HostNetworkDisabledStaysDisabled", vllmLMCache(), true, i32(0), i32(0)}, - {"HostNetworkNilReplicasUntouched", vllmLMCache(), true, nil, nil}, - {"SGLangRedisGrandfatheredScaleOutClamped", sglangLMCache(), false, i32(3), i32(1)}, - {"SGLangRedisSingletonUntouched", sglangLMCache(), false, i32(1), i32(1)}, - {"SGLangRedisDisabledStaysDisabled", sglangLMCache(), false, i32(0), i32(0)}, - {"VLLMLMCacheScalesFreely", vllmLMCache(), false, i32(3), i32(3)}, - } { - t.Run(tc.name, func(t *testing.T) { - spec := &appsv1.DeploymentSpec{ - Replicas: tc.replicas, - Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{HostNetwork: tc.hostNetwork}}, - } - clampSingletonReplicas(spec, tc.backend) - switch { - case tc.want == nil && spec.Replicas != nil: - t.Fatalf("replicas = %d, want nil", *spec.Replicas) - case tc.want != nil && spec.Replicas == nil: - t.Fatalf("replicas = nil, want %d", *tc.want) - case tc.want != nil && *spec.Replicas != *tc.want: - t.Fatalf("replicas = %d, want %d", *spec.Replicas, *tc.want) - } - }) - } -} - -// TestBuildDeploymentClampsGrandfatheredHostNetworkReplicas proves the clamp is -// wired into the render path, not merely available as a helper. -func TestBuildDeploymentClampsGrandfatheredHostNetworkReplicas(t *testing.T) { - r := &CacheBackendReconciler{} - cb := mooncakeBackend("cache", "default") - three := int32(3) - cb.Spec.Replicas = &three - - dep := r.buildDeployment(cb, &corev1.PodSpec{ - HostNetwork: true, - Containers: []corev1.Container{{Name: "master"}}, - }) - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 1 { - t.Fatalf("replicas = %v, want 1 — a grandfathered spec.replicas=3 must not schedule three masters", dep.Spec.Replicas) - } -} - -// TestHeadlessnessDiverges pins the recreate trigger in applyService. spec.clusterIP -// is immutable, so a Service can never be migrated in place between headless -// ("None") and a virtual ClusterIP in either direction — it must be recreated. An -// unassigned live value ("") means the apiserver has not allocated one yet and must -// never trigger a delete, or a transient read would churn the Service. -func TestHeadlessnessDiverges(t *testing.T) { - for _, tc := range []struct { - name string - live, desired string - want bool - }{ - {"UnassignedLiveNeverDiverges", "", corev1.ClusterIPNone, false}, - {"UnassignedLiveNeverDivergesForVirtualIP", "", "", false}, - {"VirtualIPWantsHeadless", "10.96.0.10", corev1.ClusterIPNone, true}, - {"HeadlessWantsVirtualIP", corev1.ClusterIPNone, "", true}, - {"HeadlessStaysHeadless", corev1.ClusterIPNone, corev1.ClusterIPNone, false}, - {"VirtualIPStaysVirtualIP", "10.96.0.10", "", false}, - } { - t.Run(tc.name, func(t *testing.T) { - if got := headlessnessDiverges(tc.live, tc.desired); got != tc.want { - t.Fatalf("headlessnessDiverges(%q, %q) = %v, want %v", tc.live, tc.desired, got, tc.want) - } - }) - } -} diff --git a/internal/controller/cachebackend_kvevent_gate_test.go b/internal/controller/cachebackend_kvevent_gate_test.go index abbd9a89..1f0a87ec 100644 --- a/internal/controller/cachebackend_kvevent_gate_test.go +++ b/internal/controller/cachebackend_kvevent_gate_test.go @@ -26,19 +26,9 @@ import ( // gate enabled (no opt-out annotation), used by the KV-event gate tests. The // shared lmcacheBackend fixture opts out, so gate tests use this instead. func gatedLMCacheBackend(name, ns string) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Generation: 1}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Replicas: ptrInt32(1), - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, - }, - } + cb := lmcacheBackend(name, ns) + delete(cb.Annotations, annotationRequireKVEvents) + return cb } // setFirstAvailableAt patches the backend's write-once timeout anchor @@ -332,7 +322,9 @@ func TestIntegrationKVEventReadinessGate(t *testing.T) { Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("external.example.svc:6379"), + LMCache: lmcacheBackend("fixture", ns).Spec.LMCache.DeepCopy(), + RemoteStorage: externalRedisStorage("external.example.svc:6379"), + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, }, } if err := k8s.Create(ctx, cb); err != nil { @@ -341,8 +333,8 @@ func TestIntegrationKVEventReadinessGate(t *testing.T) { reconcile(t, r, "ext", ns) got := getBackend(t, r, "ext", ns) - if got.Status.Endpoint != "external.example.svc:6379" { - t.Fatalf("endpoint = %q, want mirrored external endpoint", got.Status.Endpoint) + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "external.example.svc:6379" { + t.Fatalf("remoteStorage status = %+v, want mirrored external endpoint", got.Status.RemoteStorage) } // External never enters the KV-event gate: readiness comes from // admission accepting the endpoint (reason ExternalEndpointAccepted), @@ -438,7 +430,7 @@ func TestIntegrationKVEventGateAutoReconcileOnPollerWrite(t *testing.T) { // recorder so it runs without envtest. func TestKVEventGateEmitsTransitionEvents(t *testing.T) { cb := gatedLMCacheBackend("cache", "ns1") - r, rec := newReconcilerWithRecorder(t, cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) // Cold start: deployment not ready → no gate event yet. reconcile(t, r, "cache", "ns1") @@ -465,7 +457,7 @@ func TestKVEventGateEmitsNoKVEventsObservedOnTimeout(t *testing.T) { cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ FirstEventTimeout: &metav1.Duration{Duration: time.Second}, } - r, rec := newReconcilerWithRecorder(t, cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") // Deployment Ready (managedReadiness keys on replica counts), and the latched @@ -487,7 +479,7 @@ func TestKVEventGateEmitsNoKVEventsObservedOnTimeout(t *testing.T) { // Ready must NOT re-emit "first KV event observed". func TestKVEventGateKVEventsObservedFiresOnceAcrossRollout(t *testing.T) { cb := gatedLMCacheBackend("cache", "ns1") - r, rec := newReconcilerWithRecorder(t, cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") // First event → KVEventsObserved fires once. @@ -526,7 +518,8 @@ func TestKVEventGateKVEventsObservedFiresOnceAcrossRollout(t *testing.T) { // NOT a misleading second KVEventsObserved (events never stopped flowing). func TestKVEventGateDeploymentRecoveryIsBackendRecovered(t *testing.T) { cb := gatedLMCacheBackend("cache", "ns1") - r, rec := newReconcilerWithRecorder(t, cb) + requireRemoteStorageForReadiness(cb) + r, rec := newReconcilerWithRecorderOptions(t, true, cb) reconcile(t, r, "cache", "ns1") // Become Ready with events flowing. @@ -541,13 +534,13 @@ func TestKVEventGateDeploymentRecoveryIsBackendRecovered(t *testing.T) { // Lose replicas → Degraded (deployment cause, not KV). markDeploymentDegraded(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") - expectEvent(t, drainEvents(rec), eventReasonBackendDegraded) + expectEvent(t, drainEvents(rec), reasonRemoteStorageUnavailable) // Replicas recover; lastEventAt is still set (events never lost). markDeploymentReady(t, r, "cache", "ns1", 1) reconcile(t, r, "cache", "ns1") evs := drainEvents(rec) - expectEvent(t, evs, eventReasonBackendRecovered) + expectEvent(t, evs, reasonRemoteStorageReady) expectNoEvent(t, evs, reasonKVEventsObserved) } @@ -562,7 +555,7 @@ func TestKVEventGateRequeueDoesNotStarveMatchedPodsRefresh(t *testing.T) { cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ MatchLabels: map[string]string{"app": "vllm"}, } - r, _ := newReconcilerWithRecorder(t, cb) + r, _ := newReconcilerWithRecorderOptions(t, false, cb) r.MatchedEnginePodsRequeueInterval = 7 * time.Second reconcile(t, r, "cache", "ns1") @@ -576,8 +569,8 @@ func TestKVEventGateRequeueDoesNotStarveMatchedPodsRefresh(t *testing.T) { if err != nil { t.Fatalf("reconcile: %v", err) } - if rd := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady); rd == nil || rd.Status != metav1.ConditionFalse || rd.Reason != reasonAwaitingFirstKVEvent { - t.Fatalf("Ready = %+v, want False/AwaitingFirstKVEvent (precondition)", rd) + if rd := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady); rd == nil || rd.Status != metav1.ConditionFalse || rd.Reason != reasonNoEnginePods { + t.Fatalf("Ready = %+v, want False/NoEnginePods (connector observation wins before the KV-event gate)", rd) } if res.RequeueAfter != 7*time.Second { t.Fatalf("RequeueAfter = %s, want 7s (matched-pods cadence must win over the gate's firstEventTimeout window)", res.RequeueAfter) diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index 3fc4d27e..be647532 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -10,6 +10,7 @@ import ( "strconv" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -144,6 +145,9 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con Message: message, ObservedGeneration: backend.Generation, }) + if equality.Semantic.DeepEqual(before.Status, backend.Status) { + return + } if err := r.Status().Patch(ctx, backend, client.MergeFrom(before)); err != nil { backend.Status = before.Status log.FromContext(ctx).V(1).Info("LMCache MP connector status refresh skipped: patch failed", diff --git a/internal/controller/cachebackend_lmcache_mp_status_test.go b/internal/controller/cachebackend_lmcache_mp_status_test.go index 54a305b6..12e642aa 100644 --- a/internal/controller/cachebackend_lmcache_mp_status_test.go +++ b/internal/controller/cachebackend_lmcache_mp_status_test.go @@ -9,7 +9,6 @@ import ( "strings" "testing" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" @@ -259,49 +258,6 @@ func TestLMCacheMPReadyBase(t *testing.T) { } } -func TestTypedMPManagedRedisRestartDoesNotCascadeEngineDeployment(t *testing.T) { - backend := typedMPStatusBackend() - backend.Status.ObservedServerInstance = "legacy-remote-instance-latch" - backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{Image: "redis:7.4-alpine"}, - } - controller := true - engineDep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: "sglang-engine", Namespace: "ns1", UID: "engine-dep-uid"}, - Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "sglang"}}}}, - } - engineRS := &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{ - Name: "sglang-engine-rs", Namespace: "ns1", UID: "engine-rs-uid", - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", Kind: "Deployment", Name: engineDep.Name, UID: engineDep.UID, Controller: &controller, - }}, - }} - enginePod := typedMPStatusPod("sglang-engine-pod", true, true, true) - enginePod.OwnerReferences = []metav1.OwnerReference{{ - APIVersion: "apps/v1", Kind: "ReplicaSet", Name: engineRS.Name, UID: engineRS.UID, Controller: &controller, - }} - - r := newReconciler(newScheme(t), backend, engineDep, engineRS, enginePod) - reconcile(t, r, backend.Name, backend.Namespace) - - var gotDep appsv1.Deployment - if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns1", Name: engineDep.Name}, &gotDep); err != nil { - t.Fatalf("get engine Deployment: %v", err) - } - if got := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != "" { - t.Fatalf("Redis lifecycle cascaded engine Deployment with trigger %q", got) - } - var gotBackend cachev1alpha1.CacheBackend - if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns1", Name: backend.Name}, &gotBackend); err != nil { - t.Fatalf("get CacheBackend: %v", err) - } - if gotBackend.Status.ObservedServerInstance != "" { - t.Fatalf("typed MP retained legacy remote instance latch %q", gotBackend.Status.ObservedServerInstance) - } -} - func TestMPConditionEventsFireOnTransitionOrObservedGeneration(t *testing.T) { backend := typedMPStatusBackend() meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ diff --git a/internal/controller/cachebackend_managed.go b/internal/controller/cachebackend_managed.go index adfe49a3..1553ded3 100644 --- a/internal/controller/cachebackend_managed.go +++ b/internal/controller/cachebackend_managed.go @@ -16,12 +16,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "time" ) -// reconcileManaged renders the cache-server PodSpec + Service via the runtime -// adapter, wraps them into a Deployment + Service owned by the CR, and -// publishes the resolved endpoint to status. +// reconcileManaged receives a storage-provider PodSpec + Service, wraps them +// in controller-owned resources, and publishes the observed remote endpoint. // // Apply drives desired state; status reflects observed state. The two must not // block each other: if a desired-state write fails (e.g. a transient API-server @@ -32,10 +30,10 @@ import ( func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend, rendered *backendadapter.RenderedStorage) (ctrl.Result, error) { podSpec, svcSpec := rendered.PodSpec, rendered.Service if podSpec == nil || svcSpec == nil { - // Engine-local adapters such as native SGLang HiCache intentionally - // render no cache-server. Reuse the unmanaged lifecycle to shed any + // Engine-local adapters intentionally render no provider workload. Reuse + // the unmanaged lifecycle to shed any // previously owned workload and clear server-backed status. - logger.V(1).Info("adapter rendered no cache-server; treating as unmanaged", + logger.V(1).Info("adapter rendered no provider workload; treating as unmanaged", "namespace", backend.Namespace, "name", backend.Name) return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) } @@ -43,10 +41,8 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo dep := r.buildDeployment(backend, podSpec) svc := r.buildService(backend, svcSpec) - // Skip Service + HPA when applyDeployment failed. The HPA targets the - // Deployment by name, so running it after a foreign-ownership failure - // could scale another controller's workload; the Service is independent - // but pointless to expose alongside a Deployment we don't own. Status + // Skip the Service when applyDeployment failed. It is pointless and unsafe + // to expose pods alongside a Deployment we do not own. Status // observation still runs below (it has its own ownership guards) so the // CR isn't held hostage to apply churn. applyErr := r.applyDeployment(ctx, backend, dep) @@ -54,9 +50,6 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo if svcErr := r.applyService(ctx, backend, svc); svcErr != nil { applyErr = svcErr } - if hpaErr := r.reconcileHPA(ctx, backend, dep); hpaErr != nil && applyErr == nil { - applyErr = hpaErr - } } var live appsv1.Deployment @@ -103,87 +96,20 @@ func (r *CacheBackendReconciler) reconcileManaged(ctx context.Context, logger lo } requeueAfter, statusErr := r.updateManagedStatus(ctx, backend, endpoint, &live, applyErr == nil) - // Do NOT short-circuit on statusErr — the cascade is independent - // recovery for stale engine sockets and must not be skipped just - // because the unrelated managed-status patch (matchedEnginePods, - // Ready / Progressing / Degraded conditions, …) hit a - // transient conflict. The cascade has its own patchStatus path - // for the latch field, gated separately. Capture the error and - // return it AFTER the cascade has run. - - // Cache-server restart cascade: when the Ready cache-server pod - // SERVER-INSTANCE IDENTIFIER changes (either a pod UID swap or a - // restart-sum advance from an in-place kubelet-driven container - // restart — see currentServerInstanceID's godoc for the shape), - // cascade-restart every engine Deployment that was injected - // against this backend so they re-establish their LMCache client - // socket (the upstream LMServerConnector opens its TCP socket in - // __init__ only and silently fails every subsequent PUT with EPIPE - // after a server restart, until the engine pod itself rolls). Always - // runs (even when applyErr != nil OR updateManagedStatus errored), - // since the cascade is independent of whether THIS reconcile pass - // made a successful apply or a successful unrelated status update: - // a transient apply / status-write churn must not delay engine - // recovery from a cache-server outage. A non-zero cascadeWait means - // the rate-limit window suppressed the cascade; honor it on the - // requeue so we retry exactly at the boundary. - cascadeWait := time.Duration(0) - if isTypedLMCachePodLocal(backend) { - // In the typed MP hierarchy this managed workload is Redis L3, not the - // engine's connector endpoint. Redis failure/recovery belongs to the MP - // server's L2 adapter; rolling every engine on a Redis restart creates - // serving disruption without repairing that adapter. MP native-sidecar - // restarts are observed separately through ConnectorReady. - r.clearServerInstanceLatchShadow(backend) - if backend.Status.ObservedServerInstance != "" { - if clearErr := r.patchStatus(ctx, backend, func() { backend.Status.ObservedServerInstance = "" }); clearErr != nil && statusErr == nil { - statusErr = clearErr - } - } - } else { - cascadeWait = r.reconcileServerInstance(ctx, logger, backend) - } - if cascadeWait > 0 && (requeueAfter == 0 || cascadeWait < requeueAfter) { - requeueAfter = cascadeWait - } - // Schedule an unconditional periodic health re-poll on managed backends. - // Typed MP uses it to refresh engine-Pod native-sidecar health; legacy - // server-backed paths use it to observe the cache-server pod set. For the - // latter, an in-place container - // restart (kubelet respawning a crashed cache-server container - // without bumping pod.UID) does NOT change owned-Deployment status - // counts, and the controller deliberately does not watch Pods - // cluster-wide (see refreshMatchedEnginePods godoc). The - // matched-engine-pods cadence above does not cover this case - // either: when an operator removes spec.engineSelector after - // engines were injected, len(matchedEnginePods)→0 and that - // cadence stops firing, leaving in-place restarts unobservable - // until something unrelated triggers a reconcile. Pinning a - // floor at the rate-limit interval bounds the observation - // latency for in-place restarts at one cadence (cheap: one - // Pod List + one Deployment Get per backend per cadence). - pollCadence := r.minServerRestartCascadeInterval() - if requeueAfter == 0 || pollCadence < requeueAfter { - requeueAfter = pollCadence - } - + requeueAfter = minNonZero(requeueAfter, r.matchedEnginePodsRequeueInterval()) if applyErr != nil { // Return the error so controller-runtime's workqueue // rate-limiter requeues the reconcile. Per the // sigs.k8s.io/controller-runtime/pkg/reconcile contract, when // the error is non-nil the `Result` is ignored — including any // RequeueAfter we might set here — so there is no point - // pretending to schedule the cascade retry at the rate-limit - // boundary on this path. The rate-limiter's backoff cadence is - // the actual retry schedule; the next successful reconcile - // then re-enters the cascade path at its own boundary. + // pretending to schedule a timed retry on this path. The rate-limiter's + // backoff cadence is the actual retry schedule. return ctrl.Result{}, applyErr } if statusErr != nil { - // Surface the deferred status-write failure after the cascade - // has had its chance to recover engine FDs. Same workqueue - // rate-limiter semantics as the applyErr path: Result is - // ignored when err != nil. + // Surface the deferred status-write failure. Same workqueue rate-limiter + // semantics as the applyErr path: Result is ignored when err != nil. return ctrl.Result{}, statusErr } diff --git a/internal/controller/cachebackend_managed_test.go b/internal/controller/cachebackend_managed_test.go index 1d957e5d..4e4a9c72 100644 --- a/internal/controller/cachebackend_managed_test.go +++ b/internal/controller/cachebackend_managed_test.go @@ -9,7 +9,6 @@ import ( "errors" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,14 +25,13 @@ import ( func TestReconcileLMCacheCreatesWorkload(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) r := newReconciler(scheme, cb) reconcile(t, r, "cache", "ns1") dep := getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 2 { - t.Fatalf("deployment replicas = %v, want 2", dep.Spec.Replicas) + if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 1 { + t.Fatalf("deployment replicas = %v, want fixed managed-Redis singleton", dep.Spec.Replicas) } owner := metav1.GetControllerOf(dep) if owner == nil || owner.Kind != "CacheBackend" || owner.Name != "cache" || owner.Controller == nil || !*owner.Controller { @@ -45,17 +43,17 @@ func TestReconcileLMCacheCreatesWorkload(t *testing.T) { t.Fatalf("containers = %d, want 1", len(containers)) } c := containers[0] - if c.Name != "lmcache-server" { - t.Fatalf("container name = %q, want lmcache-server (standalone server, not the all-in-one vLLM)", c.Name) + if c.Name != "redis-l2" { + t.Fatalf("container name = %q, want redis-l2", c.Name) } if c.Image == "" { t.Fatalf("container image is empty") } - if !containsStr(c.Command, "lmcache_server") { - t.Fatalf("container command = %v, want to start with lmcache_server", c.Command) + if !containsStr(c.Args, "redis-server") { + t.Fatalf("container args = %v, want redis-server while preserving the image entrypoint", c.Args) } - if len(c.Ports) != 1 || c.Ports[0].ContainerPort != 65432 { - t.Fatalf("ports = %v, want exactly one port on 65432 (lm:// scheme)", c.Ports) + if len(c.Ports) != 1 || c.Ports[0].ContainerPort != 6379 { + t.Fatalf("ports = %v, want exactly one Redis port on 6379", c.Ports) } svc := &corev1.Service{} @@ -65,8 +63,8 @@ func TestReconcileLMCacheCreatesWorkload(t *testing.T) { if svc.Spec.Type != corev1.ServiceTypeClusterIP { t.Fatalf("service type = %q, want ClusterIP", svc.Spec.Type) } - if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 65432 { - t.Fatalf("service ports = %v, want exactly one port on 65432", svc.Spec.Ports) + if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 6379 { + t.Fatalf("service ports = %v, want exactly one port on 6379", svc.Spec.Ports) } if so := metav1.GetControllerOf(svc); so == nil || so.Name != "cache" { t.Fatalf("service controller owner = %+v, want CacheBackend/cache", so) @@ -83,19 +81,22 @@ func TestReconcileLMCacheCreatesWorkload(t *testing.T) { } updated := getBackend(t, r, "cache", "ns1") - wantEndpoint := "cache.ns1.svc.cluster.local:65432" - if updated.Status.Endpoint != wantEndpoint { - t.Fatalf("status.endpoint = %q, want %q (engine-agnostic host:port; lm:// prefix is the adapter's job)", updated.Status.Endpoint, wantEndpoint) + wantEndpoint := "cache.ns1.svc.cluster.local:6379" + if updated.Status.RemoteStorage == nil || updated.Status.RemoteStorage.Endpoint != wantEndpoint { + t.Fatalf("status.remoteStorage = %+v, want endpoint %q", updated.Status.RemoteStorage, wantEndpoint) } if updated.Status.ObservedGeneration != 1 { t.Fatalf("status.observedGeneration = %d, want 1", updated.Status.ObservedGeneration) } - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonRolloutInProgress { - t.Fatalf("Ready condition = %+v, want False/RolloutInProgress (no ready replicas yet)", cond) + if cond := findCondition(updated.Status.Conditions, conditionTypeRemoteStorageReady); cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonRemoteStoragePending { + t.Fatalf("RemoteStorageReady = %+v, want Unknown/%s (no ready replicas yet)", cond, reasonRemoteStoragePending) + } + if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonConnectorUnverified { + t.Fatalf("Ready = %+v, want Unknown/%s until an engine Pod is observed", cond, reasonConnectorUnverified) } } -func TestReconcileLegacyCacheWithTypedObservationRetainsProviderWorkload(t *testing.T) { +func TestReconcileManagedRedisWithObservationRetainsProviderWorkload(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("legacy-observed", "ns1") cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "model-a"} @@ -111,89 +112,9 @@ func TestReconcileLegacyCacheWithTypedObservationRetainsProviderWorkload(t *test t.Fatalf("managed service was not retained: %v", err) } got := getBackend(t, r, cb.Name, cb.Namespace) - wantEndpoint := "legacy-observed.ns1.svc.cluster.local:65432" - if got.Status.Endpoint != wantEndpoint { - t.Fatalf("status.endpoint = %q, want %q", got.Status.Endpoint, wantEndpoint) - } -} - -// TestReconcileManagedMooncake is the C2-reconciles-Mooncake DoD: a canonical -// Mooncake remote provider must reconcile into a managed mooncake_master -// Deployment + Service, and -// status.endpoint must be the master's RPC host:port (the engine-agnostic -// address the pod webhook later turns into mooncakestore://). The RPC port -// being first in the rendered Service is what makes serviceEndpoint resolve it. -func TestReconcileManagedMooncake(t *testing.T) { - scheme := newScheme(t) - cb := mooncakeBackend("cache", "ns1") - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - dep := getDeployment(t, r, "cache", "ns1") - owner := metav1.GetControllerOf(dep) - if owner == nil || owner.Kind != "CacheBackend" || owner.Name != "cache" { - t.Fatalf("deployment controller owner = %+v, want CacheBackend/cache", owner) - } - containers := dep.Spec.Template.Spec.Containers - if len(containers) != 1 { - t.Fatalf("containers = %d, want 1", len(containers)) - } - c := containers[0] - if c.Name != "mooncake-master" { - t.Fatalf("container name = %q, want mooncake-master", c.Name) - } - if c.Image == "" { - t.Fatalf("container image is empty") - } - if !containsStr(c.Command, "mooncake_master") { - t.Fatalf("container command = %v, want to start with mooncake_master", c.Command) - } - if len(c.Ports) == 0 || c.Ports[0].ContainerPort != 50051 { - t.Fatalf("first container port = %v, want RPC port 50051 first", c.Ports) - } - - svc := &corev1.Service{} - if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, svc); err != nil { - t.Fatalf("get service: %v", err) - } - if svc.Spec.Type != corev1.ServiceTypeClusterIP { - t.Fatalf("service type = %q, want ClusterIP", svc.Spec.Type) - } - if len(svc.Spec.Ports) == 0 || svc.Spec.Ports[0].Port != 50051 { - t.Fatalf("first service port = %v, want RPC port 50051 first (serviceEndpoint uses Ports[0])", svc.Spec.Ports) - } - - updated := getBackend(t, r, "cache", "ns1") - wantEndpoint := "cache.ns1.svc.cluster.local:50051" - if updated.Status.Endpoint != wantEndpoint { - t.Fatalf("status.endpoint = %q, want %q (master RPC host:port; mooncakestore:// prefix is the adapter's job)", updated.Status.Endpoint, wantEndpoint) - } - if updated.Status.ObservedGeneration != 1 { - t.Fatalf("status.observedGeneration = %d, want 1", updated.Status.ObservedGeneration) - } -} - -func TestReconcileCanonicalManagedMooncake(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("canonical-mooncake", "ns1") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Mooncake: &cachev1alpha1.MooncakeRemoteStorageSpec{}, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, cb.Name, cb.Namespace) - - dep := getDeployment(t, r, cb.Name, cb.Namespace) - if got := dep.Spec.Template.Spec.Containers[0].Name; got != "mooncake-master" { - t.Fatalf("container name = %q, want mooncake-master", got) - } - got := getBackend(t, r, cb.Name, cb.Namespace) - if want := "canonical-mooncake.ns1.svc.cluster.local:50051"; got.Status.Endpoint != want { - t.Fatalf("status.endpoint = %q, want %q", got.Status.Endpoint, want) + wantEndpoint := "legacy-observed.ns1.svc.cluster.local:6379" + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != wantEndpoint { + t.Fatalf("status.remoteStorage = %+v, want endpoint %q", got.Status.RemoteStorage, wantEndpoint) } } @@ -236,9 +157,11 @@ func TestReconcileLMCacheIdempotent(t *testing.T) { } } -func TestReconcileLMCacheCaseInsensitiveEngine(t *testing.T) { - // The canonical VLLM runtime must route to the managed adapter path. - for _, runtime := range []cachev1alpha1.CacheBackendRuntime{cachev1alpha1.CacheBackendRuntimeVLLM} { +func TestReconcileManagedRedisForSupportedLMCacheRuntimes(t *testing.T) { + for _, runtime := range []cachev1alpha1.CacheBackendRuntime{ + cachev1alpha1.CacheBackendRuntimeVLLM, + cachev1alpha1.CacheBackendRuntimeSGLang, + } { t.Run(string(runtime), func(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") @@ -251,8 +174,8 @@ func TestReconcileLMCacheCaseInsensitiveEngine(t *testing.T) { if err != nil { t.Fatalf("expected a managed Deployment for runtime=%q, got error: %v", runtime, err) } - if got := dep.Spec.Template.Spec.Containers[0].Name; got != "lmcache-server" { - t.Fatalf("container = %q, want lmcache-server (runtime=%q must resolve to RuntimeVLLM)", got, runtime) + if got := dep.Spec.Template.Spec.Containers[0].Name; got != "redis-l2" { + t.Fatalf("container = %q, want redis-l2 for runtime=%q", got, runtime) } }) } @@ -268,7 +191,6 @@ func TestReconcileLMCacheCaseInsensitiveEngine(t *testing.T) { func TestReconcileLMCacheConflictThenConverge(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) var conflictsRemaining int32 = 3 // first 3 Deployment Updates → 409 funcs := interceptor.Funcs{ @@ -297,7 +219,7 @@ func TestReconcileLMCacheConflictThenConverge(t *testing.T) { // (Image override mutates the managed container in-place; a no-op reconcile // would not call Update at all.) live := getBackend(t, r, "cache", "ns1") - live.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v9" + live.Spec.RemoteStorage.Redis.Image = "example.com/redis:v9" live.Generation = 2 if err := r.Update(context.Background(), live); err != nil { t.Fatalf("update CR: %v", err) @@ -310,24 +232,23 @@ func TestReconcileLMCacheConflictThenConverge(t *testing.T) { if remaining := atomic.LoadInt32(&conflictsRemaining); remaining != 0 { t.Fatalf("conflictsRemaining = %d, want 0 (RetryOnConflict should consume them)", remaining) } - if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got != "example.com/lmcache-server:v9" { - t.Fatalf("deployment image = %q, want %q (apply did not converge under conflict)", got, "example.com/lmcache-server:v9") + if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got != "example.com/redis:v9" { + t.Fatalf("deployment image = %q, want %q (apply did not converge under conflict)", got, "example.com/redis:v9") } updated := getBackend(t, r, "cache", "ns1") - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionTrue { - t.Fatalf("Ready condition = %+v, want True (CR is stuck despite Deployment being ready)", cond) + if cond := findCondition(updated.Status.Conditions, conditionTypeRemoteStorageReady); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("RemoteStorageReady = %+v, want True after conflict convergence", cond) } } // TestReconcileLMCacheEndpointHeldUntilServiceExists pins the endpoint -// invariant: Status.Endpoint must only advertise an address that corresponds +// invariant: status.remoteStorage.endpoint must only advertise an address that corresponds // to a *live* Service. When applyService is rejected on the first reconcile // (so the Service was never created), the CR must not publish the desired // endpoint — clients/gateways would route to a non-existent target. func TestReconcileLMCacheEndpointHeldUntilServiceExists(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) gr := schema.GroupResource{Group: "", Resource: "services"} funcs := interceptor.Funcs{ @@ -348,12 +269,12 @@ func TestReconcileLMCacheEndpointHeldUntilServiceExists(t *testing.T) { // Deployment is created (only Service apply was blocked), so the status // pass runs and publishes the Ready + Progressing conditions. But - // Status.Endpoint must stay empty until a live Service backs it. + // The remote-storage endpoint must stay empty until a live Service backs it. if _, err := getOptionalDeployment(t, r, "cache", "ns1"); err != nil { t.Fatalf("expected deployment to be created (only Service was blocked): %v", err) } - if got := getBackend(t, r, "cache", "ns1").Status.Endpoint; got != "" { - t.Fatalf("status.endpoint = %q, want \"\" (no live Service exists yet)", got) + if got := getBackend(t, r, "cache", "ns1").Status.RemoteStorage; got == nil || got.Endpoint != "" { + t.Fatalf("status.remoteStorage = %+v, want empty endpoint (no live Service exists yet)", got) } } @@ -370,7 +291,6 @@ func TestReconcileLMCacheEndpointHeldUntilServiceExists(t *testing.T) { func TestReconcileLMCacheDeploymentVanishedAfterApply(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) var depGetCount atomic.Int32 var armed atomic.Bool @@ -419,7 +339,6 @@ func TestReconcileLMCacheDeploymentVanishedAfterApply(t *testing.T) { func TestReconcileLMCacheDeploymentLosesOwnershipAfterApply(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) // The interceptor strips Deployment owner refs on the 2nd Get per // reconcile (the post-apply read). The 1st Get (inside applyDeployment's @@ -499,13 +418,6 @@ func TestReconcileLMCacheForeignDeploymentNoStatusLeak(t *testing.T) { }, } cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) - // Wire autoscaling so reconcileHPA would otherwise create an HPA targeting - // the same-named (foreign) Deployment. The fix must skip both Service and - // HPA applies when applyDeployment fails — running them after a - // foreign-ownership failure could scale another controller's workload or - // expose its pods through our Service. - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} r := newReconciler(scheme, cb, foreign) if _, err := r.Reconcile(context.Background(), ctrl.Request{ @@ -527,21 +439,12 @@ func TestReconcileLMCacheForeignDeploymentNoStatusLeak(t *testing.T) { if len(svcs.Items) != 0 { t.Fatalf("services = %d, want 0 (dependent applies must be skipped when applyDeployment fails)", len(svcs.Items)) } - // And no HPA, despite spec.autoscaling being set — otherwise the HPA - // would scale the foreign Deployment by name. - var hpas autoscalingv2.HorizontalPodAutoscalerList - if err := r.List(context.Background(), &hpas, client.InNamespace("ns1")); err != nil { - t.Fatalf("list HPAs: %v", err) - } - if len(hpas.Items) != 0 { - t.Fatalf("HPAs = %d, want 0 (HPA must not target a foreign Deployment)", len(hpas.Items)) - } } // TestReconcileLMCacheForeignServiceNoEndpointLeak pins the foreign-ownership // guard on the Service endpoint path: if a Service with the matching name // already exists but is owned by another controller, applyService fails -// (AlreadyOwned). Status.Endpoint must NOT advertise that foreign Service's +// (AlreadyOwned). status.remoteStorage.endpoint must NOT advertise that foreign Service's // address; clients/gateways would route to the wrong workload. func TestReconcileLMCacheForeignServiceNoEndpointLeak(t *testing.T) { scheme := newScheme(t) @@ -565,7 +468,6 @@ func TestReconcileLMCacheForeignServiceNoEndpointLeak(t *testing.T) { }, } cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) r := newReconciler(scheme, cb, foreign) if _, err := r.Reconcile(context.Background(), ctrl.Request{ @@ -573,7 +475,7 @@ func TestReconcileLMCacheForeignServiceNoEndpointLeak(t *testing.T) { }); err == nil { t.Fatalf("reconcile returned nil, want error (Service already owned by another controller)") } - if got := getBackend(t, r, "cache", "ns1").Status.Endpoint; got != "" { - t.Fatalf("status.endpoint = %q, want empty (foreign Service must not leak into status)", got) + if got := getBackend(t, r, "cache", "ns1").Status.RemoteStorage; got == nil || got.Endpoint != "" { + t.Fatalf("status.remoteStorage = %+v, want empty endpoint (foreign Service must not leak into status)", got) } } diff --git a/internal/controller/cachebackend_matched_pods_test.go b/internal/controller/cachebackend_matched_pods_test.go index 1810665e..bc92ede6 100644 --- a/internal/controller/cachebackend_matched_pods_test.go +++ b/internal/controller/cachebackend_matched_pods_test.go @@ -367,10 +367,16 @@ func TestReconcileMatchedEnginePodsWriteOnlyOnChange(t *testing.T) { t.Fatalf("after first reconcile: matchedEnginePods = %v, want 2", got) } - // Second reconcile sees an identical world. The matchedEnginePods - // writer must skip its patch entirely (count unchanged). Other - // status fields are also at steady state, so the SubResourcePatch - // counter must not advance at all. + // Typed MP status is layered: the first pass observes the remote store and + // connector independently, and the next pass folds those observations into + // Ready. Allow that bounded convergence before measuring steady state. + reconcile(t, r, "cache", "ns1") + reconcile(t, r, "cache", "ns1") + reconcile(t, r, "cache", "ns1") + firstPasses = atomic.LoadInt32(&cbStatusPatches) + + // The next reconcile sees an identical, fully converged world. The + // matchedEnginePods writer must skip its patch because the count is unchanged. reconcile(t, r, "cache", "ns1") if got := atomic.LoadInt32(&cbStatusPatches); got != firstPasses { t.Fatalf("steady-state reconcile patched CacheBackend status %d more time(s); want 0", got-firstPasses) @@ -578,8 +584,8 @@ func TestReconcileMatchedEnginePodsCoexistsWithOtherStatusWriters(t *testing.T) if got.Status.MatchedEnginePods == nil || *got.Status.MatchedEnginePods != 2 { t.Fatalf("matchedEnginePods = %v, want 2", got.Status.MatchedEnginePods) } - if got.Status.Endpoint == "" { - t.Fatalf("status.endpoint dropped — the matchedEnginePods patch should not stomp on the other status writers") + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint == "" { + t.Fatalf("status.remoteStorage.endpoint dropped — the matchedEnginePods patch should not stomp on the other status writers") } if got.Status.ObservedGeneration == 0 { t.Fatalf("status.observedGeneration = 0 — the matchedEnginePods patch should not stomp on dispatch's status writes") diff --git a/internal/controller/cachebackend_mooncake_hostnetwork_integration_test.go b/internal/controller/cachebackend_mooncake_hostnetwork_integration_test.go deleted file mode 100644 index 1223916c..00000000 --- a/internal/controller/cachebackend_mooncake_hostnetwork_integration_test.go +++ /dev/null @@ -1,293 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "testing" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" -) - -// TestIntegrationMooncakeHostNetworkAndHeadlessService exercises the Mooncake data -// plane's provisioning contract against a REAL apiserver. Two behaviors here cannot -// be covered by the fake client, which skips allocation and validation: -// -// 1. clusterIP allocation — the apiserver assigns a virtual IP unless the Service -// explicitly asks for headless, so "did None actually survive the apply?" is -// only a real question against envtest. The renderer can be perfect while -// applyService silently drops the field. -// 2. clusterIP immutability — an in-place headless migration is rejected by the -// apiserver. That rejection is precisely what applyService must sidestep by -// recreating the Service instead of updating it. -// -// If either regresses, a Mooncake backend reconciles Ready and transfers zero KV. -func TestIntegrationMooncakeHostNetworkAndHeadlessService(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, _ := startEnv(t) - r := &CacheBackendReconciler{Client: k8s, Scheme: scheme, Log: logr.Discard()} - ctx := context.Background() - - t.Run("MooncakeRendersHostNetworkPodAndHeadlessService", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, mooncakeBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - dep := getDeployment(t, r, "cache", ns) - if !dep.Spec.Template.Spec.HostNetwork { - t.Fatal("master pod is not hostNetwork; mooncake's transfer engine cannot use overlay pod IPs") - } - if got := dep.Spec.Template.Spec.DNSPolicy; got != corev1.DNSClusterFirstWithHostNet { - t.Fatalf("dnsPolicy = %q, want %q (hostNetwork must keep cluster DNS)", - got, corev1.DNSClusterFirstWithHostNet) - } - if got := dep.Spec.Strategy.Type; got != appsv1.RecreateDeploymentStrategyType { - t.Fatalf("strategy = %q, want %q (a rolling surge collides on the node's ports)", - got, appsv1.RecreateDeploymentStrategyType) - } - - // The apiserver would have allocated a virtual IP had the adapter not asked - // for headless — an assertion that is meaningless against a fake client. - svc := getService(t, r, "cache", ns) - if svc.Spec.ClusterIP != corev1.ClusterIPNone { - t.Fatalf("svc.Spec.ClusterIP = %q, want %q — a virtual IP forwards only the declared ports and strands mooncake's dynamic ones", - svc.Spec.ClusterIP, corev1.ClusterIPNone) - } - }) - - t.Run("LMCacheKeepsPodNetworkAndVirtualClusterIP", func(t *testing.T) { - // Blast radius: the portable, non-privileged default must not move. - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - if dep := getDeployment(t, r, "cache", ns); dep.Spec.Template.Spec.HostNetwork { - t.Fatal("lmcache server became hostNetwork; it must stay on the pod network") - } - svc := getService(t, r, "cache", ns) - if svc.Spec.ClusterIP == corev1.ClusterIPNone || svc.Spec.ClusterIP == "" { - t.Fatalf("lmcache svc.Spec.ClusterIP = %q, want an apiserver-allocated virtual IP", svc.Spec.ClusterIP) - } - }) - - t.Run("MigratesExistingDeploymentOntoHostNetworkAndRecreate", func(t *testing.T) { - // applyDeployment overwrites the whole Spec only on CREATE; on UPDATE it - // reconciles a hand-picked subset. A Mooncake backend provisioned before this - // fix therefore owns an overlay Deployment with the API-server-defaulted - // RollingUpdate strategy, and both must migrate — otherwise the upgrade is a - // silent no-op: the master stays unreachable for the mesh, and a rolling - // surge would collide on the node's ports. - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, mooncakeBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - // Rewind the live object to its pre-fix shape. - dep := getDeployment(t, r, "cache", ns) - dep.Spec.Template.Spec.HostNetwork = false - dep.Spec.Template.Spec.DNSPolicy = corev1.DNSClusterFirst - dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RollingUpdateDeploymentStrategyType} - if err := k8s.Update(ctx, dep); err != nil { - t.Fatalf("simulate pre-fix Deployment: %v", err) - } - // Precondition: the apiserver populates a rollingUpdate block, which is - // exactly what would reject a naive .Type-only flip to Recreate. - if pre := getDeployment(t, r, "cache", ns); pre.Spec.Strategy.RollingUpdate == nil { - t.Fatal("precondition: expected an apiserver-populated rollingUpdate block") - } - - reconcile(t, r, "cache", ns) - - got := getDeployment(t, r, "cache", ns) - if !got.Spec.Template.Spec.HostNetwork { - t.Fatal("existing Deployment did not migrate onto hostNetwork; the fix would be a no-op on upgrade") - } - if want := corev1.DNSClusterFirstWithHostNet; got.Spec.Template.Spec.DNSPolicy != want { - t.Fatalf("dnsPolicy = %q, want %q after migration", got.Spec.Template.Spec.DNSPolicy, want) - } - if got.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType { - t.Fatalf("strategy = %q, want %q after migration", got.Spec.Strategy.Type, appsv1.RecreateDeploymentStrategyType) - } - if got.Spec.Strategy.RollingUpdate != nil { - t.Fatal("stale rollingUpdate block not cleared; the apiserver rejects it alongside Recreate") - } - }) - - t.Run("RevertsToPodNetworkAndRollingUpdateWhenTypeChangesAwayFromMooncake", func(t *testing.T) { - // The migration must be symmetric. Reconciling only "unset -> explicit" - // would strand a backend that switches away from Mooncake on Recreate and - // ClusterFirstWithHostNet forever, because those fields would never be - // written back to their defaults. - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, mooncakeBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - if dep := getDeployment(t, r, "cache", ns); !dep.Spec.Template.Spec.HostNetwork { - t.Fatal("precondition: Mooncake master should start on hostNetwork") - } - - cb := getBackend(t, r, "cache", ns) - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - } - if err := k8s.Update(ctx, cb); err != nil { - t.Fatalf("switch backend from Mooncake to LMCache server: %v", err) - } - reconcile(t, r, "cache", ns) - - got := getDeployment(t, r, "cache", ns) - if got.Spec.Template.Spec.HostNetwork { - t.Fatal("hostNetwork not cleared after switching away from Mooncake") - } - if want := corev1.DNSClusterFirst; got.Spec.Template.Spec.DNSPolicy != want { - t.Fatalf("dnsPolicy = %q, want %q (stale host-net policy left behind)", - got.Spec.Template.Spec.DNSPolicy, want) - } - if got.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType { - t.Fatalf("strategy = %q, want %q (Deployment stranded on Recreate)", - got.Spec.Strategy.Type, appsv1.RollingUpdateDeploymentStrategyType) - } - - // The Service must migrate too — headless -> virtual ClusterIP is the same - // immutable-field problem in reverse, and only a real apiserver both rejects - // the in-place update and allocates the replacement VIP. The reconcile above - // deletes the divergent Service; the next one recreates it. - var gone corev1.Service - switch err := k8s.Get(ctx, types.NamespacedName{Name: "cache", Namespace: ns}, &gone); { - case err == nil: - t.Fatalf("headless Service survived the switch (clusterIP %q); it must be recreated with a virtual IP", - gone.Spec.ClusterIP) - case !apierrors.IsNotFound(err): - t.Fatalf("get service: %v", err) - } - - reconcile(t, r, "cache", ns) - svc := getService(t, r, "cache", ns) - if svc.Spec.ClusterIP == corev1.ClusterIPNone || svc.Spec.ClusterIP == "" { - t.Fatalf("recreated svc.Spec.ClusterIP = %q, want an apiserver-allocated virtual IP", svc.Spec.ClusterIP) - } - }) - - t.Run("GrandfatheredScaleOutIsClampedAndItsHPARemoved", func(t *testing.T) { - // Admission rejects spec.replicas>1 and spec.autoscaling for Mooncake, but - // ValidateUpdate only rejects violations an edit *introduces*: an object - // written before that rule existed (or before its backend moved onto host - // networking) sits in etcd carrying both and is never re-validated. Rendering - // it faithfully would schedule several masters — contending for the same node - // ports, or splitting the store as independent masters. The reconciler is the - // last line of defense. The apiserver would refuse to persist such a CR now, - // so the grandfathered object is simulated in memory. - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, mooncakeBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - ghost := getBackend(t, r, "cache", ns).DeepCopy() - three := int32(3) - ghost.Spec.Replicas = &three - ghost.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 5} - - // An HPA had already scaled the live Deployment out before the upgrade. - live := getDeployment(t, r, "cache", ns) - live.Spec.Replicas = &three - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("simulate HPA-scaled Deployment: %v", err) - } - - desired := r.buildDeployment(ghost, live.Spec.Template.Spec.DeepCopy()) - if err := r.applyDeployment(ctx, ghost, desired); err != nil { - t.Fatalf("applyDeployment: %v", err) - } - got := getDeployment(t, r, "cache", ns) - if got.Spec.Replicas == nil || *got.Spec.Replicas != 1 { - t.Fatalf("replicas = %v, want 1 — the clamp must override both spec.replicas and the HPA-preserved live value", - got.Spec.Replicas) - } - - // ...and no HPA may survive to undo the clamp on the next scaling decision. - if err := r.reconcileHPA(ctx, ghost, desired); err != nil { - t.Fatalf("reconcileHPA: %v", err) - } - var hpa autoscalingv2.HorizontalPodAutoscaler - switch err := k8s.Get(ctx, types.NamespacedName{Name: "cache", Namespace: ns}, &hpa); { - case err == nil: - t.Fatal("an HPA exists for a hostNetwork master; it would fight the singleton clamp and could split the store") - case !apierrors.IsNotFound(err): - t.Fatalf("get hpa: %v", err) - } - }) - - t.Run("RecreatesServiceStuckOnAnImmutableVirtualClusterIP", func(t *testing.T) { - // A Mooncake backend provisioned before this fix owns a Service carrying an - // allocated virtual IP. clusterIP is immutable, so an in-place update can - // never make it headless: applyService must delete it and let the next - // reconcile recreate it. Without this the upgrade is a no-op and the backend - // stays Ready while transferring nothing. - ns := freshNS(t, k8s) - cb := mooncakeBackend("cache", ns) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - if err := k8s.Get(ctx, types.NamespacedName{Name: "cache", Namespace: ns}, cb); err != nil { - t.Fatalf("get CacheBackend (for owner ref UID): %v", err) - } - - stale := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: ns}, - Spec: corev1.ServiceSpec{ - // No ClusterIP set -> the apiserver allocates a virtual IP. - Type: corev1.ServiceTypeClusterIP, - Selector: selectorLabels("cache"), - Ports: []corev1.ServicePort{{Name: "rpc", Port: 50051, Protocol: corev1.ProtocolTCP}}, - }, - } - if err := controllerutil.SetControllerReference(cb, stale, scheme); err != nil { - t.Fatalf("set controller reference: %v", err) - } - if err := k8s.Create(ctx, stale); err != nil { - t.Fatalf("create pre-fix service: %v", err) - } - if stale.Spec.ClusterIP == corev1.ClusterIPNone || stale.Spec.ClusterIP == "" { - t.Fatalf("precondition: pre-fix Service should carry an allocated virtual IP, got %q", stale.Spec.ClusterIP) - } - - // First pass must delete the divergent Service rather than fail forever on - // the immutable field. - reconcile(t, r, "cache", ns) - var gone corev1.Service - switch err := k8s.Get(ctx, types.NamespacedName{Name: "cache", Namespace: ns}, &gone); { - case err == nil: - t.Fatalf("pre-fix Service still present with clusterIP %q; applyService must delete it", gone.Spec.ClusterIP) - case !apierrors.IsNotFound(err): - t.Fatalf("get service: %v", err) - } - - // The next pass recreates it headless. - reconcile(t, r, "cache", ns) - svc := getService(t, r, "cache", ns) - if svc.Spec.ClusterIP != corev1.ClusterIPNone { - t.Fatalf("recreated svc.Spec.ClusterIP = %q, want %q", svc.Spec.ClusterIP, corev1.ClusterIPNone) - } - }) -} diff --git a/internal/controller/cachebackend_mp_lifecycle_test.go b/internal/controller/cachebackend_mp_lifecycle_test.go new file mode 100644 index 00000000..d3ba43c2 --- /dev/null +++ b/internal/controller/cachebackend_mp_lifecycle_test.go @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) + +func TestReconcileManagedRedisCreatesSingletonWorkload(t *testing.T) { + backend := lmcacheBackend("cache", "ns1") + reconciler := newReconciler(newScheme(t), backend) + + reconcile(t, reconciler, backend.Name, backend.Namespace) + + deployment := getDeployment(t, reconciler, backend.Name, backend.Namespace) + if deployment.Spec.Replicas == nil || *deployment.Spec.Replicas != 1 { + t.Fatalf("managed Redis replicas = %v, want 1", deployment.Spec.Replicas) + } + if len(deployment.Spec.Template.Spec.Containers) != 1 || deployment.Spec.Template.Spec.Containers[0].Name != "redis-l2" { + t.Fatalf("managed Redis containers = %+v", deployment.Spec.Template.Spec.Containers) + } + var service corev1.Service + if err := reconciler.Get(context.Background(), types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace}, &service); err != nil { + t.Fatalf("get managed Redis Service: %v", err) + } + if len(service.Spec.Ports) != 1 || service.Spec.Ports[0].Port != 6379 { + t.Fatalf("managed Redis Service ports = %+v", service.Spec.Ports) + } + + got := getBackend(t, reconciler, backend.Name, backend.Namespace) + wantEndpoint := "cache.ns1.svc.cluster.local:6379" + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Provider != cachev1alpha1.CacheBackendRemoteStorageProviderRedis || got.Status.RemoteStorage.Endpoint != wantEndpoint { + t.Fatalf("remote-storage status = %+v, want Redis endpoint %q", got.Status.RemoteStorage, wantEndpoint) + } +} + +func TestReconcileExternalRedisCreatesNoWorkload(t *testing.T) { + backend := lmcacheBackend("external", "ns1") + backend.Spec.RemoteStorage = externalRedisStorage("redis.example:6379") + reconciler := newReconciler(newScheme(t), backend) + + reconcile(t, reconciler, backend.Name, backend.Namespace) + + assertNoManagedWorkload(t, reconciler, backend.Name, backend.Namespace) + got := getBackend(t, reconciler, backend.Name, backend.Namespace) + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "redis.example:6379" || got.Status.RemoteStorage.Ready != metav1.ConditionTrue { + t.Fatalf("external Redis status = %+v", got.Status.RemoteStorage) + } + ready := findCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionUnknown || ready.Reason != reasonConnectorUnverified { + t.Fatalf("Ready = %+v, want Unknown/%s until an injected engine Pod is observed", ready, reasonConnectorUnverified) + } +} + +func TestReconcileHostOnlyMPHasNoProviderWorkload(t *testing.T) { + backend := lmcacheBackend("host-only", "ns1") + backend.Spec.RemoteStorage = nil + reconciler := newReconciler(newScheme(t), backend) + + reconcile(t, reconciler, backend.Name, backend.Namespace) + + assertNoManagedWorkload(t, reconciler, backend.Name, backend.Namespace) + got := getBackend(t, reconciler, backend.Name, backend.Namespace) + if got.Status.RemoteStorage != nil { + t.Fatalf("host-only backend published remote-storage status: %+v", got.Status.RemoteStorage) + } +} + +func assertNoManagedWorkload(t *testing.T, reconciler *CacheBackendReconciler, name, namespace string) { + t.Helper() + key := types.NamespacedName{Name: name, Namespace: namespace} + if err := reconciler.Get(context.Background(), key, &appsv1.Deployment{}); !apierrors.IsNotFound(err) { + t.Fatalf("Deployment lookup error = %v, want NotFound", err) + } + if err := reconciler.Get(context.Background(), key, &corev1.Service{}); !apierrors.IsNotFound(err) { + t.Fatalf("Service lookup error = %v, want NotFound", err) + } +} diff --git a/internal/controller/cachebackend_probe.go b/internal/controller/cachebackend_probe.go index c46cc439..6f254186 100644 --- a/internal/controller/cachebackend_probe.go +++ b/internal/controller/cachebackend_probe.go @@ -118,9 +118,7 @@ func init() { // to zero for every label combination. Package-private so tests in this // package can assert on per-test counts without leaking state across // runs; intentionally not exported because production callers have no -// reason to zero an operator-visible metric. Mirrors the helper next to -// backendServerRestartCascadesTotal in cachebackend_server_restart.go -// (the package convention for controller-runtime-registered counters). +// reason to zero an operator-visible metric. func resetProbeResultMetricForTest() { probeResultMetric.Reset() } diff --git a/internal/controller/cachebackend_probe_integration_test.go b/internal/controller/cachebackend_probe_integration_test.go index 3497d949..1d199450 100644 --- a/internal/controller/cachebackend_probe_integration_test.go +++ b/internal/controller/cachebackend_probe_integration_test.go @@ -256,9 +256,10 @@ func TestIntegrationFunctionalProbeGate(t *testing.T) { cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "lm://test.example.com:9999", + Endpoint: "test.example.com:6379", + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } if err := k8s.Patch(ctx, cb, client.MergeFrom(before)); err != nil { t.Fatalf("patch to external ownership: %v", err) diff --git a/internal/controller/cachebackend_reconciler.go b/internal/controller/cachebackend_reconciler.go index 1e637b85..3a9897af 100644 --- a/internal/controller/cachebackend_reconciler.go +++ b/internal/controller/cachebackend_reconciler.go @@ -11,7 +11,6 @@ import ( adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -28,7 +27,7 @@ import ( // between otherwise-unrelated reconcile triggers. The reconciler does not // Watch Pods by design (see refreshMatchedEnginePods godoc); without a // self-requeue, the count would only refresh when the CR, the owned -// Deployment, Service, or HPA changed. 30s strikes a balance between +// Deployment or Service changed. 30s strikes a balance between // operator responsiveness and reconcile pressure on a large fleet. Tests // override via the `MatchedEnginePodsRequeueInterval` reconciler field to // avoid baking the 30s delay into the suite. @@ -95,20 +94,6 @@ type CacheBackendReconciler struct { // sync.Map inside is usable from struct construction — the rate-limit // gate works on the first reconcile without explicit initialization. probeLimiter probeRateLimiter - - // MinServerRestartCascadeInterval overrides the rate-limit window for - // the cache-server restart cascade. Zero means "use - // [DefaultMinServerRestartCascadeInterval]". Production wiring leaves - // this zero; envtest / unit tests shrink the window to keep per-test - // runtime cheap. - MinServerRestartCascadeInterval time.Duration - - // serverInstanceCascade tracks the last cascade-restart time per - // backend so the rate-limit window is enforced in-process. Lazily - // initialized in SetupWithManager AND defensively in - // reconcileServerInstance (the latter so unit tests that bypass - // SetupWithManager get a working reconciler). - serverInstanceCascade *serverInstanceCascade } // probeRateLimit returns the effective rate-limit for the functional-probe @@ -148,16 +133,13 @@ func (r *CacheBackendReconciler) matchedEnginePodsChurnRequeueInterval() time.Du // +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch -// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch -// Reconcile drives a CacheBackend toward its desired state. External backends -// only mirror their configured endpoint to status; managed backends (LMCache -// in Phase 1) ask the registered runtime adapter for the cache-server pod -// spec + service spec, wrap them into a Deployment + Service the controller -// owns, optionally reconcile an HPA from spec.autoscaling, and publish the -// resolved endpoint. +// Reconcile drives a CacheBackend toward its desired state. External Redis +// bindings mirror their configured endpoint to remote-storage status; managed +// Redis bindings render a singleton Deployment and Service. LMCache connector +// health is derived independently from the selected engine Pods. // // On every reconcile — including ones that return an apply error — transitions // in the observed Ready condition (entering/leaving Ready=False/ @@ -250,8 +232,7 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request // SetupWithManager sets up the controller with the Manager. Owns(Deployment) // guarantees that a child's status flipping (e.g. AvailableReplicas dropping // to zero) re-triggers a Reconcile so emitTransitionEvents observes the -// change; the HPA is owned so the controller re-reconciles when the -// autoscaler updates spec.replicas or its own status. +// change. func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { if r.Recorder == nil { r.Recorder = mgr.GetEventRecorder("cachebackend-controller") @@ -264,9 +245,6 @@ func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { // passes Client, not APIReader). r.APIReader = mgr.GetAPIReader() } - if r.serverInstanceCascade == nil { - r.serverInstanceCascade = newServerInstanceCascade() - } return ctrl.NewControllerManagedBy(mgr). // NOTE: DO NOT add a predicate that filters status-only updates here. // The KV-event readiness gate depends on the CacheIndex poller's @@ -277,6 +255,5 @@ func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&cachev1alpha1.CacheBackend{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). - Owns(&autoscalingv2.HorizontalPodAutoscaler{}). Complete(r) } diff --git a/internal/controller/cachebackend_reconciler_test.go b/internal/controller/cachebackend_reconciler_test.go index e5a7904f..c10bd9a5 100644 --- a/internal/controller/cachebackend_reconciler_test.go +++ b/internal/controller/cachebackend_reconciler_test.go @@ -13,6 +13,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -54,11 +55,6 @@ func newReconciler(scheme *runtime.Scheme, objs ...client.Object) *CacheBackendR Client: c, Scheme: scheme, Log: logr.Discard(), - // Seed a real serverInstanceCascade so lifecycle tests that - // assert on the in-process shadow / lastAt / counted maps - // actually exercise the clear path rather than skipping - // the check on a nil pointer. - serverInstanceCascade: newServerInstanceCascade(), } configureTestRegistries(r) return r @@ -68,9 +64,7 @@ func configureTestRegistries(r *CacheBackendReconciler) { if r.Registry != nil && r.BackendRegistry != nil { return } - registries := builtinadapters.New(builtinadapters.Options{ - LMCacheServerImage: "lmcache/standalone:test", - }) + registries := builtinadapters.New(builtinadapters.Options{}) if r.Registry == nil { r.Registry = registries.Runtime } @@ -84,11 +78,12 @@ func setupTestCacheBackendReconciler(mgr ctrl.Manager, r *CacheBackendReconciler return r.SetupWithManager(mgr) } -func externalLMCacheStorage(endpoint string) *cachev1alpha1.CacheBackendRemoteStorageSpec { +func externalRedisStorage(endpoint string) *cachev1alpha1.CacheBackendRemoteStorageSpec { return &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: endpoint, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } } @@ -104,7 +99,7 @@ func reconcile(t *testing.T, r *CacheBackendReconciler, name, namespace string) func ptrInt32(v int32) *int32 { return &v } -// lmcacheBackend is the shared managed-backend fixture. It opts OUT of the +// lmcacheBackend is the shared typed MP fixture with managed Redis. It opts OUT of the // KV-event readiness gate via the inferencecache.io/require-kv-events: // "false" annotation so the many tests that assert rollout-driven Ready / // Degraded conditions, HPA behavior, apply-error status, and transition @@ -122,11 +117,27 @@ func lmcacheBackend(name, namespace string) *cachev1alpha1.CacheBackend { Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{ + Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + }, + }, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, }, } } @@ -156,28 +167,6 @@ func getBackend(t *testing.T, r *CacheBackendReconciler, name, namespace string) return &cb } -// mooncakeBackend is the managed-Mooncake fixture, mirroring lmcacheBackend: -// it opts OUT of the KV-event readiness gate so the rollout-driven Ready -// assertion is orthogonal to the gate. -func mooncakeBackend(name, namespace string) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Generation: 1, - Annotations: map[string]string{"inferencecache.io/require-kv-events": "false"}, - }, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - }, - } -} - func volumeNames(vs []corev1.Volume) []string { names := make([]string, len(vs)) for i := range vs { @@ -212,10 +201,9 @@ func newReconcilerWithInterceptor(scheme *runtime.Scheme, funcs interceptor.Func WithInterceptorFuncs(funcs). Build() r := &CacheBackendReconciler{ - Client: c, - Scheme: scheme, - Log: logr.Discard(), - serverInstanceCascade: newServerInstanceCascade(), + Client: c, + Scheme: scheme, + Log: logr.Discard(), } configureTestRegistries(r) return r diff --git a/internal/controller/cachebackend_resources_integration_test.go b/internal/controller/cachebackend_resources_integration_test.go index bad10197..37532bdd 100644 --- a/internal/controller/cachebackend_resources_integration_test.go +++ b/internal/controller/cachebackend_resources_integration_test.go @@ -34,11 +34,6 @@ func TestIntegrationCacheBackendResources(t *testing.T) { newCanonicalBackend := func(namespace string) *cachev1alpha1.CacheBackend { cb := lmcacheBackend("cache", namespace) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - } return cb } @@ -53,9 +48,9 @@ func TestIntegrationCacheBackendResources(t *testing.T) { reconcile(t, r, "cache", ns) cb := getBackend(t, r, "cache", ns) - if cb.Spec.RemoteStorage.LMCacheServer.Resources != nil { - t.Fatalf("renderer default leaked into spec.remoteStorage.lmCacheServer.resources: %+v", - cb.Spec.RemoteStorage.LMCacheServer.Resources) + if cb.Spec.RemoteStorage.Redis.Resources != nil { + t.Fatalf("renderer default leaked into spec.remoteStorage.redis.resources: %+v", + cb.Spec.RemoteStorage.Redis.Resources) } wantReq := resource.MustParse("4Gi") @@ -75,7 +70,7 @@ func TestIntegrationCacheBackendResources(t *testing.T) { // rendered container MUST reflect it byte-for-byte. ns := freshNS(t, k8s) cb := newCanonicalBackend(ns) - cb.Spec.RemoteStorage.LMCacheServer.Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("12Gi"), }, @@ -99,4 +94,27 @@ func TestIntegrationCacheBackendResources(t *testing.T) { t.Fatalf("container Limits[memory] = %v, want operator-supplied %v", got.String(), wantLim.String()) } }) + + t.Run("ManagedWorkloadSchedulingHonored", func(t *testing.T) { + ns := freshNS(t, k8s) + cb := newCanonicalBackend(ns) + grace := int64(45) + cb.Spec.RemoteStorage.Workload = &cachev1alpha1.CacheBackendManagedWorkloadSpec{ + NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + ServiceAccountName: "cache-provider", + TerminationGracePeriodSeconds: &grace, + } + if err := k8s.Create(ctx, cb); err != nil { + t.Fatalf("create CacheBackend: %v", err) + } + reconcile(t, r, "cache", ns) + + pod := getDeployment(t, r, "cache", ns).Spec.Template.Spec + if pod.NodeSelector["kubernetes.io/os"] != "linux" || pod.ServiceAccountName != "cache-provider" { + t.Fatalf("managed workload scheduling not rendered: %+v", pod) + } + if pod.TerminationGracePeriodSeconds == nil || *pod.TerminationGracePeriodSeconds != 45 { + t.Fatalf("terminationGracePeriodSeconds = %v, want 45", pod.TerminationGracePeriodSeconds) + } + }) } diff --git a/internal/controller/cachebackend_schema_trim_integration_test.go b/internal/controller/cachebackend_schema_trim_integration_test.go index dc43d172..e886efd7 100644 --- a/internal/controller/cachebackend_schema_trim_integration_test.go +++ b/internal/controller/cachebackend_schema_trim_integration_test.go @@ -6,6 +6,7 @@ package controller import ( "context" + "strings" "testing" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -111,22 +112,45 @@ func TestIntegrationCacheBackendSchemaTrim(t *testing.T) { }) } + for _, provider := range []string{"LMCacheServer", "Mooncake"} { + t.Run("reject-provider-"+provider, func(t *testing.T) { + u := newManaged("reject-provider-" + strings.ToLower(provider)) + if err := unstructured.SetNestedMap(u.Object, map[string]any{ + "provider": provider, "ownership": "Managed", + }, "spec", "remoteStorage"); err != nil { + t.Fatalf("set spec.remoteStorage: %v", err) + } + if err := c.Create(ctx, u); !apierrors.IsInvalid(err) { + t.Fatalf("create with remoteStorage.provider=%q error = %v, want Invalid from CRD enum", provider, err) + } + }) + } + // Removed spec fields are pruned on create and never round-trip. obj is a // separate RFC-1123 object name (the field name is mixed-case and cannot be // used as metadata.name). specCases := []struct { - name string - obj string - path []string + name string + obj string + path []string + value any }{ - {"lookupTimeoutMs", "retired-spec-lookup-timeout", []string{"spec", "integration", "lookupTimeoutMs"}}, - {"minimumPrefixTokens", "retired-spec-min-prefix-tokens", []string{"spec", "integration", "minimumPrefixTokens"}}, + {"lookupTimeoutMs", "retired-spec-lookup-timeout", []string{"spec", "integration", "lookupTimeoutMs"}, int64(7)}, + {"minimumPrefixTokens", "retired-spec-min-prefix-tokens", []string{"spec", "integration", "minimumPrefixTokens"}, int64(7)}, + {"deploymentKind", "retired-deployment-kind", []string{"spec", "deploymentKind"}, "Deployment"}, + {"replicas", "retired-replicas", []string{"spec", "replicas"}, int64(1)}, + {"autoscaling", "retired-autoscaling", []string{"spec", "autoscaling"}, map[string]any{"maxReplicas": int64(2)}}, + {"template", "retired-template", []string{"spec", "template"}, map[string]any{"schedulerName": "default-scheduler"}}, + {"hostMemory", "retired-host-memory", []string{"spec", "lmCache", "hostMemory"}, map[string]any{"capacity": "1Gi"}}, + {"workerImage", "retired-worker-image", []string{"spec", "lmCache", "workerImage"}, "example.invalid/worker:v1"}, + {"workerPort", "retired-worker-port", []string{"spec", "lmCache", "workerPort"}, int64(5555)}, + {"remoteSerde", "retired-remote-serde", []string{"spec", "lmCache", "remoteSerde"}, "cachegen"}, } for _, tc := range specCases { t.Run(tc.name, func(t *testing.T) { name := tc.obj u := newManaged(name) - if err := unstructured.SetNestedField(u.Object, int64(7), tc.path...); err != nil { + if err := unstructured.SetNestedField(u.Object, tc.value, tc.path...); err != nil { t.Fatalf("set %s: %v", tc.name, err) } if err := c.Create(ctx, u); err != nil { @@ -138,23 +162,32 @@ func TestIntegrationCacheBackendSchemaTrim(t *testing.T) { }) } - // Removed status field is pruned too. status.* needs the status subresource + // Removed status fields are pruned too. status.* needs the status subresource // to round-trip at all, so write it via a status update and confirm it does // not persist. - t.Run("indexEntries", func(t *testing.T) { - name := "retired-status-indexentries" - if err := c.Create(ctx, newManaged(name)); err != nil { - t.Fatalf("create: %v", err) - } - cur := get(name) - if err := unstructured.SetNestedField(cur.Object, int64(7), "status", "indexEntries"); err != nil { - t.Fatalf("set status.indexEntries: %v", err) - } - if err := c.Status().Update(ctx, cur); err != nil { - t.Fatalf("status update: %v", err) - } - if _, found, _ := unstructured.NestedFieldNoCopy(get(name).Object, "status", "indexEntries"); found { - t.Fatalf("status.indexEntries persisted; want pruned (field removed from schema)") - } - }) + for _, fieldName := range []string{"indexEntries", "endpoint", "observedServerInstance"} { + t.Run(fieldName, func(t *testing.T) { + name := "retired-status-indexentries" + if fieldName != "indexEntries" { + name = "retired-status-" + strings.ToLower(fieldName) + } + if err := c.Create(ctx, newManaged(name)); err != nil { + t.Fatalf("create: %v", err) + } + cur := get(name) + value := any("removed") + if fieldName == "indexEntries" { + value = int64(7) + } + if err := unstructured.SetNestedField(cur.Object, value, "status", fieldName); err != nil { + t.Fatalf("set status.%s: %v", fieldName, err) + } + if err := c.Status().Update(ctx, cur); err != nil { + t.Fatalf("status update: %v", err) + } + if _, found, _ := unstructured.NestedFieldNoCopy(get(name).Object, "status", fieldName); found { + t.Fatalf("status.%s persisted; want pruned (field removed from schema)", fieldName) + } + }) + } } diff --git a/internal/controller/cachebackend_server_restart.go b/internal/controller/cachebackend_server_restart.go deleted file mode 100644 index 0fcfe90e..00000000 --- a/internal/controller/cachebackend_server_restart.go +++ /dev/null @@ -1,1290 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "fmt" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - "github.com/cachebox-project/inference-cache/internal/enginebinding" -) - -// AnnotationCacheServerRestartTrigger is patched onto an engine Deployment's -// spec.template.metadata.annotations to drive a rolling restart of its pods -// when the controller observes that this backend's cache-server pod has been -// replaced. The value is the same server-instance identifier the controller -// writes to status.observedServerInstance (`:` for a -// single-replica backend, or a comma-joined lex-sorted list of those for a -// multi-replica backend), so two quick replacements in succession produce -// distinct annotation values (and distinct rollout revisions). Modeled on -// the kubectl rollout restart pattern -// (kubectl.kubernetes.io/restartedAt), but project-namespaced so an operator -// running both does not see the two trample each other. -// -// Why this annotation triggers a restart: the Deployment controller watches -// spec.template for any change and creates a new ReplicaSet whenever the -// template content (including its annotations) differs from the latest live -// ReplicaSet. The annotation is otherwise inert; it carries no semantics -// beyond "this pod template has been bumped". Annotating the *pod* directly -// would have no effect — the Deployment controller does not reconcile its -// children's annotations, only the template's. -const AnnotationCacheServerRestartTrigger = "inferencecache.io/cache-server-restart-trigger" - -// DefaultMinServerRestartCascadeInterval bounds how frequently the -// reconciler will cascade-restart engine Deployments in response to -// cache-server pod replacements, per CacheBackend. It dampens restart -// storms when the cache-server pod is flapping (e.g. crash-loop under -// memory pressure): every restart cascades all engines, so a -// crash-looping cache server would otherwise roll the engine fleet every -// few seconds. 30s is long enough that one full engine rollout is well -// underway before the next cascade is allowed, and short enough that a -// genuine single-restart recovery is not noticeably delayed. -// -// The window is enforced in-memory on the reconciler (see -// serverInstanceCascade.canCascade). A controller restart resets the -// rate-limit window for every backend — the in-memory `lastAt` map is -// lost — which is the intended behavior: the first cascade after -// restart is allowed immediately without waiting up to 30s. Whether -// any cascade actually fires after restart still depends on the -// normal decision (currentID differs from the durable -// status.observedServerInstance baseline AND the convergence / -// strict-superset rules); the restart does not by itself force a -// cascade on every backend. -const DefaultMinServerRestartCascadeInterval = 30 * time.Second - -// cascadeRestartReasonServerInstanceChanged is the metric label value -// used whenever the cache-server SERVER-INSTANCE IDENTIFIER differs -// from the value last persisted to status.observedServerInstance. -// This covers both kinds of "the LMCache process is fresh" transition: -// a pod UID swap (replacement, eviction, image roll) AND a restart- -// sum-only advance from an in-place kubelet-driven container restart -// (OOM with restartPolicy=Always reuses pod.UID but resets the -// process). See currentServerInstanceID for the identifier shape. It -// is the only reason today; future non-instance-change triggers -// (e.g. an operator-initiated "force cascade" surface) would add -// their own value. Kept as a constant so the metric label set is -// stable and grep-discoverable. -const cascadeRestartReasonServerInstanceChanged = "server_instance_changed" - -// backendServerRestartCascadesTotal counts cascade-restart DECISIONS -// the controller has emitted (NOT raw cache-server pod restarts, and -// NOT engine-Deployment annotate-patch operations). The counter -// advances exactly once per logical cascade event: -// -// - after the rate-limit window has elapsed for this backend -// (DefaultMinServerRestartCascadeInterval), and -// - after the engine-pod scan + Deployment-annotate phase has -// completed without error. -// -// The increment fires BEFORE the subsequent status patch: by the -// time we get here the engines are already annotated and ready to -// roll, so the metric reflects the recovery the moment it begins — -// the operator-visible counter does NOT lag behind a transient -// status-write failure. Double-counting on retry is prevented by -// the `counted` map in serverInstanceCascade: subsequent -// reconciles for the same (key, currentID) call -// shouldIncrementCascade, which returns false because the pair has -// already been counted. -// -// A cascade with ZERO matched engine Deployments still counts as one -// event — operators want flapping-server symptoms visible even when -// no engines happen to be injected today (e.g. before the engine -// fleet has been deployed, or while the operator is in the middle -// of rewiring spec.engineSelector and matchedEnginePods is empty). -// -// A crash-looping cache-server pod that restarts ten times within -// one cascade window still produces exactly one increment, because -// the rate limit collapses repeated observations into a single -// cascade per window. For raw restart rate, operators should -// compose this metric with the engine fleet's re-roll latency or -// the cache-server pod restartCount metric from kube-state-metrics; -// this Counter is the controller's record of "how many times did I -// decide to emit recovery for this backend", not "how many times -// did the cache-server crash". -// -// Partitioned by namespaced CacheBackend identity and a short reason -// code. Registered into the controller-runtime metrics registry on -// package init so it appears on the manager's /metrics endpoint (no -// per-Service registry — see internal/server/metrics.go for the -// other-direction posture). Safe to mutate concurrently; tests -// reset its inner state via resetBackendServerRestartCascadesTotalForTest. -var backendServerRestartCascadesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "inferencecache_backend_server_restart_cascades_total", - Help: "Cumulative count of cache-server-restart cascades issued by the CacheBackend controller. A single cascade re-annotates every injected engine Deployment for the backend at once; the count is incremented once per cascade, regardless of how many Deployments were touched (zero is a valid cascade — see status.observedServerInstance docs). NOT a raw restart count: rate-limiting collapses repeated observations within one cadence window into a single cascade, so this metric undercounts a flapping cache-server's restart rate by design. Partitioned by the CacheBackend's namespace + name and a short reason code.", - }, - []string{"namespace", "backend", "reason"}, -) - -func init() { - ctrlmetrics.Registry.MustRegister(backendServerRestartCascadesTotal) -} - -// resetBackendServerRestartCascadesTotalForTest resets the cascade -// counter to zero for every label combination. Package-private so -// tests in this package can assert on per-test counts without leaking -// state across runs; intentionally not exported because production -// callers have no reason to zero an operator-visible metric. -func resetBackendServerRestartCascadesTotalForTest() { - backendServerRestartCascadesTotal.Reset() -} - -// cascadeKey keys the per-backend rate-limit map. Includes the -// CacheBackend's metadata.uid alongside namespace/name so a -// delete-recreate-with-same-name does not inherit the deleted -// object's throttle window — the new backend gets a fresh first -// cascade. Without the UID, an operator who deletes and re-creates -// a CacheBackend while a cascade was still inside the rate-limit -// window would silently delay the new backend's first real -// observation by up to MinServerRestartCascadeInterval. -type cascadeKey struct { - namespace string - name string - uid string -} - -// serverInstanceCascade tracks per-backend in-process state for the -// server-restart cascade: rate-limiting timestamps AND a shadow of -// the most-recently-attempted observedServerInstance value. Used in- -// process by CacheBackendReconciler; a process restart clears it -// (intentional — see DefaultMinServerRestartCascadeInterval). -// -// Why the shadow exists: status.observedServerInstance is written via -// patchStatus, which can fail (conflict / transient apiserver error). -// If the patch silently failed and a real cache-server replacement -// happened before a later successful patch, prior would read as "" -// on the next reconcile and the replacement would be misclassified -// as a first observation (empty→set: no cascade), stranding the -// engines on stale sockets. The shadow records every currentID the -// reconciler attempted to persist so the next reconcile can recover -// the intended baseline even when status is still empty. Authority -// order: a non-empty status.observedServerInstance ALWAYS wins (it's -// the durable on-cluster source of truth); the shadow is only -// consulted when status is empty (controller restart loses the -// shadow but rebuilds it on first observation, so the worst-case -// degenerates to "treat as first observation"). Process-restart -// risk is acceptable because the K8s-resident status field carries -// the value across restarts in the steady-state path. -// -// The maps are not actively pruned: entries are bounded by the -// number of distinct CacheBackends ever observed by the running -// process, each entry costs ~128 bytes, and a typical cluster has -// at most a few hundred CacheBackends over its lifetime. Operator- -// driven churn in the thousands-per-process would warrant adding a -// TTL-based prune; not worth the complexity for the expected scale. -type serverInstanceCascade struct { - mu sync.Mutex - lastAt map[cascadeKey]time.Time - // shadow records the last currentID the reconciler ATTEMPTED to - // persist into status.observedServerInstance (whether or not the - // patch succeeded). Read fallback when the status field is empty. - shadow map[cascadeKey]string - // counted records the most recent currentID we have already - // incremented the cascade counter for. Lets the cascade-fired - // branch advance the metric exactly once per logical cascade - // EVENT (identified by (key, currentID)), even when the post- - // annotate status patch takes multiple retries: the first - // attempt records the (key, currentID) and Inc()s; subsequent - // reconciles for the same (key, currentID) see counted == - // currentID and skip the increment. Separate from `shadow` - // because a "baseline" or "converged-superset" persist also - // updates the shadow but must NOT be counted. - counted map[cascadeKey]string - // cleared records that the latch for this key was explicitly - // cleared (a lifecycle exit from the managed path — - // reconcileExternal or reconcileUnmanaged — wiped the shadow - // and asked the reconciler to publish - // status.observedServerInstance=""). - // The sentinel survives a transient status-patch failure on - // that clear: the in-memory cleared bit overrides any stale - // non-empty status field on the NEXT reconcile, so a - // managed→External→managed flip in a tight patch-failure - // window cannot misclassify the new period's first Ready pod - // as a replacement of the prior-period identifier. - // recordAttempt clears this flag (a new baseline is being - // persisted, so we are no longer in the "explicitly cleared" - // state). - cleared map[cascadeKey]bool -} - -func newServerInstanceCascade() *serverInstanceCascade { - return &serverInstanceCascade{ - lastAt: map[cascadeKey]time.Time{}, - shadow: map[cascadeKey]string{}, - counted: map[cascadeKey]string{}, - cleared: map[cascadeKey]bool{}, - } -} - -// recordAttempt stamps the most recent currentID the reconciler -// decided to persist for the given key. Called BEFORE the patch -// attempt so a subsequent reconcile can recover the intended -// baseline even if the patch fails. Also clears the "explicitly -// cleared" sentinel (a new baseline is being recorded, so the -// post-clear gap is closed for this key). -func (s *serverInstanceCascade) recordAttempt(key cascadeKey, currentID string) { - s.mu.Lock() - defer s.mu.Unlock() - s.shadow[key] = currentID - delete(s.cleared, key) -} - -// lastAttempt returns the most recent currentID the reconciler -// decided to persist for this key, or "" if no attempt has been -// recorded since process start. Used as the fallback prior when -// status.observedServerInstance is empty. -func (s *serverInstanceCascade) lastAttempt(key cascadeKey) string { - s.mu.Lock() - defer s.mu.Unlock() - return s.shadow[key] -} - -// shouldIncrementCascade reports whether the cascade counter has -// already been advanced for (key, currentID). If not — i.e. this -// is the first observation of this (key, currentID) since process -// start or since the last clear() — records the pair and returns -// true; the caller is then responsible for Inc()'ing the metric. -// Subsequent calls for the same pair return false, even when the -// post-annotate status patch is retrying across reconciles. This -// is what enforces "one increment per cascade EVENT" against the -// retry-after-failed-persist path. -func (s *serverInstanceCascade) shouldIncrementCascade(key cascadeKey, currentID string) bool { - s.mu.Lock() - defer s.mu.Unlock() - if s.counted[key] == currentID { - return false - } - s.counted[key] = currentID - return true -} - -// clear drops the rate-limit timestamp, the shadow baseline, and -// the counted-cascade ledger for the given key, AND records an -// explicit "cleared" sentinel that overrides any stale non-empty -// status.observedServerInstance on the next reconcile. Called when -// the backend transitions out of the managed path (External, -// unsupported runtime) and its -// status.observedServerInstance is also cleared on the cluster. -// -// The sentinel exists because the in-memory clear runs BEFORE the -// status patch that wipes the cluster-resident field, and that -// patch can fail. Without the sentinel, a tight failure window -// (clear shadow → status patch fails → operator flips back to -// managed before retry) would leave shadow empty + status holding -// the prior period's identifier; the reconciler would then read -// prior = statusField (stale) and misclassify the first new -// managed-period Ready pod as a replacement, triggering an -// unnecessary engine cascade despite the documented "inert/cleared" -// contract. The sentinel overrides that: cleared[key]=true means -// "treat prior as empty regardless of what statusField says". -// recordAttempt deletes the sentinel (a new baseline is now being -// persisted; we are no longer in the cleared state). Controller -// restart loses the sentinel; the External/Unmanaged -// path's patchStatus retry on subsequent reconciles is the durable -// backstop for that case. -func (s *serverInstanceCascade) clear(key cascadeKey) { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.lastAt, key) - delete(s.shadow, key) - delete(s.counted, key) - s.cleared[key] = true -} - -// isCleared reports whether the latch for this key has been -// explicitly cleared via clear() and not yet recorded a new -// attempt. Used by reconcileServerInstance to override a stale -// non-empty status field on the next managed-period reconcile. -func (s *serverInstanceCascade) isCleared(key cascadeKey) bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.cleared[key] -} - -// canCascade reports whether enough time has elapsed since the previous -// cascade for the given backend. If true, it stamps now as the most -// recent cascade time. If false, it returns the remaining wait until the -// next cascade is allowed so the caller can RequeueAfter exactly that -// long. The check + stamp are done under one lock so concurrent -// reconciles of the same backend do not double-cascade. -func (s *serverInstanceCascade) canCascade(key cascadeKey, now time.Time, window time.Duration) (bool, time.Duration) { - s.mu.Lock() - defer s.mu.Unlock() - last, ok := s.lastAt[key] - if !ok { - s.lastAt[key] = now - return true, 0 - } - elapsed := now.Sub(last) - if elapsed >= window { - s.lastAt[key] = now - return true, 0 - } - return false, window - elapsed -} - -// clearServerInstanceLatchShadow wipes the in-memory shadow + rate- -// limit timestamp for this backend. Called from lifecycle paths that -// intentionally clear the on-cluster status.observedServerInstance -// field (reconcileExternal, reconcileUnmanaged). -// The shadow must follow the cluster-visible field; otherwise a -// later managed→External→managed transition in -// the same controller process would consult the lingering shadow, -// resolve effectivePrior to the stale prior-period value, and -// misclassify the first new Ready pod as a replacement — -// triggering an unnecessary engine cascade even though the -// documented contract says the latch is "cleared/inert" between -// managed periods. -// -// Safe to call before serverInstanceCascade has been lazy-inited -// (no-op in that case). -func (r *CacheBackendReconciler) clearServerInstanceLatchShadow(backend *cachev1alpha1.CacheBackend) { - if r.serverInstanceCascade == nil { - return - } - r.serverInstanceCascade.clear(cascadeKey{ - namespace: backend.Namespace, - name: backend.Name, - uid: string(backend.UID), - }) -} - -// minServerRestartCascadeInterval returns the effective rate-limit window -// for this reconciler — the per-reconciler override, or -// DefaultMinServerRestartCascadeInterval when unset. -func (r *CacheBackendReconciler) minServerRestartCascadeInterval() time.Duration { - if r.MinServerRestartCascadeInterval > 0 { - return r.MinServerRestartCascadeInterval - } - return DefaultMinServerRestartCascadeInterval -} - -// reconcileServerInstance observes the current Ready cache-server -// instance identifier for this managed backend and, on a change that -// reflects an actual cache-server replacement (a pod that was Ready -// before is no longer Ready, or a container inside a persisting pod -// has restarted), cascade-restarts every injected engine Deployment -// by patching AnnotationCacheServerRestartTrigger onto each -// Deployment's pod template. Status is patched on every transition; -// cascading is rate-limited per CacheBackend (see -// DefaultMinServerRestartCascadeInterval) and the function returns a -// non-zero requeue when the rate-limit deferred the cascade so the -// caller's reconcile result schedules the retry exactly at the window -// boundary. -// -// "Transient" transitions that do NOT cascade: -// - empty → set (first observation; there is no prior server-instance -// to invalidate, so by definition no engine sockets are stale — -// any engines that connected during the "" window connected to the -// very pod we are now baselining) -// - prior set strictly grows AND the owning Deployment is still -// rolling (a maxSurge midpoint: the old pod is still Ready while -// the new one comes up; the subsequent transition that drops the -// old pod IS a cascade). When the Deployment has converged at the -// wider count instead — operator-driven scale-up — the widened -// set IS persisted as the new baseline, so a later replacement of -// any of the added pods cascades correctly. -// -// When no Ready cache-server pod exists at all (currentID = ""), the -// reconciler leaves status.observedServerInstance at its prior value -// rather than clearing it. The latch is intentionally stale-while- -// unavailable: a transient cache-server outage (Deployment scaled to -// 0, all pods Terminating mid-rollout, image pull stuck) must NOT -// look like "everything is fine, no instance" to the next reconcile, -// because the eventual recovery will bring back a fresh-UID pod set -// and that transition IS a real cache-server replacement that -// requires cascading. Clearing the latch in the no-Ready window -// would lose the prior-set memory and turn the recovery's -// "" → "new-uid:0" transition into a first-observation baseline -// (no cascade) — exactly the scenario this whole controller exists -// to prevent. The latch returns to a current-view value as soon as -// a Ready pod is observed. -// -// "Real" transitions that DO cascade: -// - any prior pod is no longer in the current set (pod replaced) -// - any persisting pod's restart-count sum advanced (in-place -// container restart) -// -// Fail-soft: every error path (server-instance observation — -// owned Deployment Get, ReplicaSet owner-chain Get, Pod list; -// engine-cascade observation/annotate; status patch) logs at V(1) -// and returns a positive requeue duration (typically -// minServerRestartCascadeInterval) rather than escalating to the -// caller as a Reconcile error. Cascading is best-effort recovery -// from a known soft-failure mode; a transient apiserver hiccup must -// not back off the rest of the reconcile, but it also must not -// silently strand recovery — the requeue ensures the next reconcile -// retries within the cascade window. -func (r *CacheBackendReconciler) reconcileServerInstance(ctx context.Context, logger logr.Logger, backend *cachev1alpha1.CacheBackend) time.Duration { - currentID, converged, err := r.currentServerInstanceID(ctx, backend) - if err != nil { - // A transient observation failure (owned Deployment Get, - // ReplicaSet owner-chain Get, or Pod list — see - // currentServerInstanceID for the chain) leaves us unable - // to decide whether a cascade is needed. Return the rate- - // limit interval as the requeue hint so the reconcile - // retries within the same window we'd cascade in — without - // this, the only path back is unrelated watch events, which - // can leave the recovery stranded (especially in the - // selector-removed-but-still-injected case). - logger.V(1).Info("server-restart cascade skipped: server-instance observation failed", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - if currentID == "" { - // No Ready cache-server pod yet — nothing to anchor to. - return 0 - } - - if r.serverInstanceCascade == nil { - // Defensive lazy-init; the manager wiring should set this in - // SetupWithManager but unit tests construct the reconciler - // directly and skip that path. - r.serverInstanceCascade = newServerInstanceCascade() - } - key := cascadeKey{ - namespace: backend.Namespace, - name: backend.Name, - uid: string(backend.UID), - } - - // Compute effective prior. Authority order: cleared sentinel → - // shadow → statusField. - // - // 1) The "cleared" sentinel is checked first. A lifecycle exit - // from the managed path (External / Unmanaged) - // sets it via clear(); it survives a transient status-patch - // failure on the same clear. If the operator flips back to - // managed before the External-side patchStatus retry has - // landed, statusField would still hold the prior-period - // identifier — but cleared==true forces effectivePrior="", - // so the next observation is treated as a clean first-set - // (no false cascade against the stale value). - // - // 2) The in-memory shadow IS the authority when set: it records - // the last currentID the reconciler decided to persist, - // whether or not the patch landed. statusField is the durable - // projection of that shadow — when they disagree, it is - // because the most recent persist failed and has not yet - // retried. Trusting statusField over a non-empty shadow would - // re-introduce the round-21 regression: a converged scale-up - // persist that fails would leave shadow="A:0,B:0" while - // status still held the pre-scale "A:0"; a subsequent - // replacement of just the added pod ("A:0,C:0") would look - // like a strict superset of the status-derived "A:0" and miss - // the cascade. - // - // 3) Otherwise (cold start, controller restart), statusField is - // the durable source — the K8s API survived the restart even - // though the in-process state did not. - statusField := backend.Status.ObservedServerInstance - var prior string - switch { - case r.serverInstanceCascade.isCleared(key): - prior = "" - case r.serverInstanceCascade.lastAttempt(key) != "": - prior = r.serverInstanceCascade.lastAttempt(key) - default: - prior = statusField - } - if prior == currentID { - // Logical state is in sync. If the K8s status field is also - // in sync, we are done. If the field is empty because a prior - // persist failed (shadow holds the value but status does - // not), retry the patch idempotently so the operator-visible - // status field eventually reflects the shadow. Do NOT - // re-cascade and do NOT re-increment the counter — the - // recovery already happened on the original observation. - if statusField == currentID { - return 0 - } - if err := r.patchStatus(ctx, backend, func() { - backend.Status.ObservedServerInstance = currentID - }); err != nil { - logger.V(1).Info("server-restart cascade: status-field patch retry failed (shadow holds the in-process baseline; will retry)", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - return 0 - } - - // Empty → set: first observation. There is no prior server- - // instance to invalidate, so engines that connected during the - // empty window are connecting to the very pod we are baselining. - // Persist as the baseline and stop. Record the attempt FIRST so - // a patch failure does not lose the baseline (see shadow godoc). - if prior == "" { - r.serverInstanceCascade.recordAttempt(key, currentID) - if err := r.patchStatus(ctx, backend, func() { - backend.Status.ObservedServerInstance = currentID - }); err != nil { - logger.V(1).Info("server-restart cascade: initial observedServerInstance patch failed (in-memory shadow retains the baseline for the next reconcile)", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - return 0 - } - - // Distinguish a real cache-server replacement from a transient - // rolling-update widening (old + new pod both Ready briefly). - // - // Strict-superset transitions split into two cases by the owning - // Deployment's convergence flag (see currentServerInstanceID): - // - // - NOT converged (rolling-update midpoint): do NOT persist the - // widened set. If we did, a failed rollout that gets rolled - // back (new pod briefly Ready, then killed by failing - // readiness, leaving the original pod alone) would later look - // like "the new pod was replaced" and false-cascade — but the - // original cache-server process and existing engine sockets - // never changed. Keeping the prior latch intact makes the - // rollback path a true no-op. - // - // - Converged (steady-state scale-up): persist the widened set - // as the new baseline. The operator raised spec.replicas, the - // Deployment has reached steady state at the higher count, and - // the added pods are real cache-server processes. If we did - // not persist, a later replacement of just one of the added - // pods would still be a strict superset of the pinned prior - // map — instanceChangeRequiresCascade would return false and - // no cascade would fire, leaving engines that connected to the - // replaced pod with stale sockets. Record the attempt FIRST - // so a patch failure does not lose the widened baseline. - if !instanceChangeRequiresCascade(prior, currentID) { - if converged { - r.serverInstanceCascade.recordAttempt(key, currentID) - if err := r.patchStatus(ctx, backend, func() { - backend.Status.ObservedServerInstance = currentID - }); err != nil { - logger.V(1).Info("server-restart cascade: converged-superset observedServerInstance patch failed (in-memory shadow retains the widened baseline)", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - logger.V(1).Info("server-restart cascade skipped: superset at Deployment steady state (scale-up); latch advanced", - "namespace", backend.Namespace, "name", backend.Name, - "prior", prior, "current", currentID) - return 0 - } - logger.V(1).Info("server-restart cascade skipped: transient rolling-update superset", - "namespace", backend.Namespace, "name", backend.Name, - "prior", prior, "current", currentID) - return 0 - } - - // Rate-limit: a cascade per backend is allowed at most once per - // minServerRestartCascadeInterval. While the window is open we - // neither cascade nor advance status.observedServerInstance — - // leaving the prior value pinned guarantees the next eligible - // reconcile still sees the change and tries again. The key - // includes the CacheBackend's UID so a delete-recreate under the - // same name gets a fresh window. - ok, wait := r.serverInstanceCascade.canCascade(key, time.Now(), r.minServerRestartCascadeInterval()) - if !ok { - logger.V(1).Info("server-restart cascade deferred: rate-limited", - "namespace", backend.Namespace, "name", backend.Name, - "prior", prior, "current", currentID, "retryAfter", wait.String()) - return wait - } - - count, err := r.cascadeRestartEngineDeployments(ctx, backend, currentID) - if err != nil { - // Soft-fail: log and keep going. The next reconcile (or the - // matched-pods cadence requeue) will retry. Do NOT advance - // status.observedServerInstance — leaving it on the prior value - // keeps the change visible to the retry. - logger.V(1).Info("server-restart cascade: engine Deployment list/patch failed", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - logger.V(1).Info("server-restart cascade: engine Deployments annotated", - "namespace", backend.Namespace, "name", backend.Name, - "prior", prior, "current", currentID, "deployments", count) - - // Increment the cascade counter exactly once per cascade EVENT, - // keyed by (cascadeKey, currentID). shouldIncrementCascade - // records the pair atomically and returns false on subsequent - // calls for the same pair; this enforces the "one increment per - // cascade event" contract even when the post-annotate status - // patch takes multiple retries to land. The counter advances - // HERE (not after patchStatus succeeds) because the recovery has - // already happened — every injected engine Deployment is - // annotated, ready to roll. Subsequent retries via the shadow - // short-circuit at the top of this function do not re-enter the - // cascade path, so the metric stays in sync with the recovery. - if r.serverInstanceCascade.shouldIncrementCascade(key, currentID) { - backendServerRestartCascadesTotal.WithLabelValues(backend.Namespace, backend.Name, cascadeRestartReasonServerInstanceChanged).Inc() - } - - // Persist the new baseline. Record the attempt in the shadow - // FIRST so a patch failure does not lose the baseline (see the - // shadow godoc on serverInstanceCascade): on the next reconcile, - // the shadow short-circuit branch at the top will retry the - // patch idempotently without re-incrementing the counter (the - // counted map already holds this currentID). - r.serverInstanceCascade.recordAttempt(key, currentID) - if err := r.patchStatus(ctx, backend, func() { - backend.Status.ObservedServerInstance = currentID - }); err != nil { - logger.V(1).Info("server-restart cascade: observedServerInstance patch failed (annotates already issued; shadow + counted retain the event so the next reconcile retries persist without re-counting)", - "namespace", backend.Namespace, "name", backend.Name, "error", err.Error()) - return r.minServerRestartCascadeInterval() - } - return 0 -} - -// instanceChangeRequiresCascade reports whether the prior→current -// transition reflects an actual cache-server replacement that warrants -// rolling the engine fleet, as opposed to a transient -// rolling-update widening (old + new pod both Ready briefly). -// -// Algorithm: parse both strings into (pod-UID → restart-sum) maps. A -// cascade is required iff some pod-UID in prior is either absent in -// current OR has a different restart-sum in current. A current that -// is a strict superset of prior (same UIDs at same restart counts, -// plus extras) is the rolling-update midpoint — no cascade yet. -// -// The caller MUST NOT persist a strict-superset current as the new -// baseline. If it did, a rolled-back rolling update (the new pod -// briefly Ready, then killed by failing readiness, leaving the -// original pod alone) would later look like "the new pod is gone → -// real replacement" and false-cascade. Keeping the latch pinned to -// prior through superset transitions makes the rollback path a -// no-op and the genuine completion path (original pod drops) a -// correctly-detected replacement. -// -// Conservative fallback: if `prior` is non-empty but fails to parse -// (operator hand-edited the field, or a value from a prior schema -// shape survives an upgrade), force a cascade. We cannot reason about -// what the prior set was, so treating any change as a real cascade is -// safer than silently skipping it. -func instanceChangeRequiresCascade(prior, current string) bool { - pm := parseInstanceMap(prior) - if len(pm) == 0 && prior != "" { - return true - } - cm := parseInstanceMap(current) - for uid, priorSum := range pm { - curSum, ok := cm[uid] - if !ok { - return true // prior pod is gone — replacement happened. - } - if curSum != priorSum { - return true // pod persists but container restarted in place. - } - } - return false -} - -// parseInstanceMap parses the ":,:" identifier -// shape currentServerInstanceID emits into a map keyed by pod-UID. -// Malformed segments are skipped (defensive — the controller is the -// sole writer, so it should never produce bad shapes, but tolerating -// them keeps the cascade decision well-defined if the field is hand- -// edited by an operator). -func parseInstanceMap(s string) map[string]int32 { - if s == "" { - return nil - } - parts := strings.Split(s, ",") - out := make(map[string]int32, len(parts)) - for _, p := range parts { - i := strings.LastIndexByte(p, ':') - if i <= 0 || i == len(p)-1 { - continue - } - sum, err := strconv.ParseInt(p[i+1:], 10, 32) - if err != nil { - continue - } - out[p[:i]] = int32(sum) - } - return out -} - -// currentServerInstanceID returns a stable identifier representing -// the current set of Ready cache-server pods for the backend, the -// owning Deployment's convergence flag (true when the Deployment -// controller has reached steady state — `spec.replicas` == -// `status.readyReplicas` == `status.updatedReplicas` and -// `status.observedGeneration` >= `metadata.generation`), or "" when no -// Ready pod exists yet. The candidate set is the owned Deployment's -// pods — pods whose controller-owning ReplicaSet is controller-owned -// by the backend-owned Deployment, identified by both name AND UID so -// a foreign Ready pod that happens to carry the same controller- -// managed labels (or a stale ownerRef name that resolves to a -// different live object) cannot advance observedServerInstance and -// spuriously trigger an engine rollout. -// -// The identifier shape is `:` per Ready pod, -// comma-joined and lex-sorted by pod name; for a single-replica -// backend this is one segment, for multi-replica ephemeral backends -// it's a comma-joined list. The restart-sum half (sum of -// pod.status.containerStatuses[].RestartCount) makes in-place -// container restarts observable: an OOM-killed cache-server container -// respawned in the same pod reuses pod.UID, so a UID-only identifier -// would miss it — engines would keep their stale LMCache sockets. -// Including the restart-sum advances the identifier whenever the -// LMCache server process inside the pod is fresh. -// -// The convergence flag lets the caller distinguish a rolling-update -// midpoint (NOT converged: maxSurge has briefly widened the Ready set -// above target) from a steady-state scale-up (converged: the operator -// raised replicas and the new pods are part of the steady state). -// reconcileServerInstance persists a strict-superset baseline only -// when converged is true; otherwise it could pin a transient midpoint -// that a rollback would later misread as a real replacement. -// -// Reads via APIReader (uncached) where possible to avoid registering a -// Pod informer; the controller's design explicitly rejects watching -// all pods cluster-wide (see refreshMatchedEnginePods godoc). Falls -// back to the cached client when APIReader is nil (test wiring). -func (r *CacheBackendReconciler) currentServerInstanceID(ctx context.Context, backend *cachev1alpha1.CacheBackend) (string, bool, error) { - reader := client.Reader(r.APIReader) - if reader == nil { - reader = r.Client - } - - // Fetch the owned Deployment so we can authenticate candidate pods - // against its UID. Verify the live Deployment is still controlled by - // THIS CacheBackend before using it as the ownership anchor — name - // reuse / race conditions could otherwise let a foreign Deployment - // re-created under the same name be treated as ours. NotFound is the - // cold-start case (CR exists, the reconciler hasn't created the - // Deployment yet, or it was deleted out-of-band): no pods can be - // authoritatively attributed so report "no instance". - var ownedDep appsv1.Deployment - if err := reader.Get(ctx, types.NamespacedName{Namespace: backend.Namespace, Name: backend.Name}, &ownedDep); err != nil { - if apierrors.IsNotFound(err) { - return "", false, nil - } - return "", false, fmt.Errorf("get owned cache-server deployment: %w", err) - } - if !metav1.IsControlledBy(&ownedDep, backend) { - // Foreign Deployment sharing the backend's name. Refuse to - // attribute its pods to this CacheBackend. - return "", false, nil - } - - matcher := labels.SelectorFromSet(selectorLabels(backend.Name)) - var pods corev1.PodList - if err := reader.List(ctx, &pods, - client.InNamespace(backend.Namespace), - client.MatchingLabelsSelector{Selector: matcher}, - ); err != nil { - return "", false, fmt.Errorf("list cache-server pods: %w", err) - } - // Build the set of container names the owned Deployment renders. - // containerRunSum sums restart counts ONLY for these — sidecars - // injected by other admission webhooks (Istio's istio-proxy, - // linkerd's linkerd-proxy, Datadog agents, etc.) are added to the - // pod's containerStatuses but are absent from the Deployment - // template, so including them would cascade-restart every engine - // any time a service-mesh sidecar crash-looped — completely - // unrelated to the LMCache server's actual state. - cacheServerContainers := make(map[string]struct{}, len(ownedDep.Spec.Template.Spec.Containers)) - for _, c := range ownedDep.Spec.Template.Spec.Containers { - cacheServerContainers[c.Name] = struct{}{} - } - - // Collect every Ready, attributable pod's identifier. A pod that - // is mid-rollout (Pending / Terminating) does not represent a - // serving instance — including it would let a rollout's transient - // state trigger a cascade even though the prior instance is still - // serving. A pod that is Ready but not transitively controller- - // owned by THIS backend's Deployment is rejected — see the godoc - // above for why the ownership check is required. - // - // The per-pod identifier is : where - // containerRunSum is the sum of restart counts across the - // cache-server's own containers (filtered by the owned - // Deployment's template names). pod.UID alone is invariant across - // in-place container restarts (kubelet restarting a crashed - // container reuses the pod object), but the LMCache server - // process inside that pod is fresh and every engine still holds - // a stale socket. Including the restart-count sum makes that - // case observable. - type readyPod struct { - name string - id string - } - ready := make([]readyPod, 0, len(pods.Items)) - for i := range pods.Items { - p := &pods.Items[i] - if p.DeletionTimestamp != nil { - continue - } - if !podIsReady(p) { - continue - } - owned, err := r.podOwnedByDeployment(ctx, reader, p, &ownedDep) - if err != nil { - return "", false, err - } - if !owned { - continue - } - ready = append(ready, readyPod{ - name: p.Name, - id: fmt.Sprintf("%s:%d", p.UID, containerRunSum(p, cacheServerContainers)), - }) - } - - // Deployment convergence: spec.replicas == status.readyReplicas - // == status.updatedReplicas == len(live Ready pods), AND the - // controller has observed the current generation. When all four - // hold, the Deployment is in steady state and a strict-superset - // Ready set against the prior baseline reflects a legitimate - // scale-up (not a maxSurge midpoint). All four are required: - // - readyReplicas == spec.replicas → right number of Ready pods - // (rules out maxSurge widening above target) - // - updatedReplicas == spec.replicas → all Ready pods run the - // latest revision (rules out maxSurge mid-rollout where the - // extras are the new revision) - // - observedGeneration >= metadata.generation → the controller - // has seen the current spec (rules out a just-changed spec - // whose effects haven't propagated yet) - // - len(live Ready pods) == spec.replicas → the live pod list - // we just took for the identifier MATCHES the Deployment - // status's claim. Without this clause, a stale Deployment - // status (the apps controller hasn't yet observed the - // maxSurge new pod) could report readyReplicas==1 while we - // observed 2 Ready pods in the same reconcile — the - // status counters would lie convergence even though the - // midpoint is genuinely transient, and we would persist the - // widened latch as a "scale-up" baseline. A later rollback - // dropping the new pod would then look like a real - // replacement (a UID disappeared from the latch) and - // false-cascade. Cross-checking against the live count is - // cheap and closes that race. - // nil spec.Replicas defaults to 1 per the Deployment defaulter - // (kubebuilder/apiserver default), so we collapse nil to 1. - wantReplicas := int32(1) - if ownedDep.Spec.Replicas != nil { - wantReplicas = *ownedDep.Spec.Replicas - } - converged := ownedDep.Status.ObservedGeneration >= ownedDep.Generation && - ownedDep.Status.ReadyReplicas == wantReplicas && - ownedDep.Status.UpdatedReplicas == wantReplicas && - int32(len(ready)) == wantReplicas - - if len(ready) == 0 { - return "", converged, nil - } - sort.Slice(ready, func(i, j int) bool { return ready[i].name < ready[j].name }) - ids := make([]string, len(ready)) - for i := range ready { - ids[i] = ready[i].id - } - return strings.Join(ids, ","), converged, nil -} - -// containerRunSum returns the sum of restart counts across the cache- -// server's own containers, scoped to the set of container names from -// the owned Deployment's pod template. Used to detect in-place -// restarts of the cache-server container (kubelet respawning a crashed -// container reuses the pod UID but increments restartCount). Sidecars -// outside the owned set are excluded — see currentServerInstanceID's -// godoc for why. init / ephemeral containers are also excluded -// (RestartCount is on containerStatuses, not init/ephemeral surfaces). -func containerRunSum(pod *corev1.Pod, cacheServerContainers map[string]struct{}) int32 { - var sum int32 - for i := range pod.Status.ContainerStatuses { - cs := &pod.Status.ContainerStatuses[i] - if _, ok := cacheServerContainers[cs.Name]; !ok { - continue - } - sum += cs.RestartCount - } - return sum -} - -// podOwnedByDeployment reports whether pod is transitively controller- -// owned (pod → ReplicaSet → Deployment) by the given Deployment, -// matched on both name AND UID at every link. The (owned, err) split -// distinguishes "definitively not owned" (owned=false, err=nil — bare -// pod / non-Deployment / different Deployment) from "couldn't decide" -// (err != nil — transient apiserver/RBAC issue). Callers MUST -// propagate the error so currentServerInstanceID does not advance -// the latch over an incomplete picture; collapsing a transient -// ReplicaSet Get failure to "not owned" would shrink the identifier -// and look like a pod replacement, triggering a false cascade. -// -// A NotFound on the ReplicaSet is treated as "not owned" (the chain -// has been GCd) — the pod cannot be authoritatively attributed. -func (r *CacheBackendReconciler) podOwnedByDeployment(ctx context.Context, reader client.Reader, pod *corev1.Pod, dep *appsv1.Deployment) (bool, error) { - rsRef := metav1.GetControllerOf(pod) - if rsRef == nil || rsRef.Kind != "ReplicaSet" || !ownerRefIsAppsV1(rsRef) { - return false, nil - } - var rs appsv1.ReplicaSet - if err := reader.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: rsRef.Name}, &rs); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("get owning ReplicaSet %s/%s: %w", pod.Namespace, rsRef.Name, err) - } - if rs.UID != rsRef.UID { - return false, nil - } - depRef := metav1.GetControllerOf(&rs) - if depRef == nil || depRef.Kind != "Deployment" || !ownerRefIsAppsV1(depRef) { - return false, nil - } - return depRef.Name == dep.Name && depRef.UID == dep.UID, nil -} - -// ownerRefIsAppsV1 reports whether the owner reference points at -// apps/v1. OwnerReference.apiVersion is a required field, so a strict -// equality match is the right shape — an empty value is invalid input -// that we should reject rather than tolerate. -func ownerRefIsAppsV1(ref *metav1.OwnerReference) bool { - return ref.APIVersion == "apps/v1" -} - -// podIsReady reports whether the pod is in the Running phase with its -// Ready condition True. Mirrors the readiness signal kubelet writes into -// the pod's status, which is what the Deployment controller uses to -// decide whether to count a pod toward AvailableReplicas. We deliberately -// do not require a grace period beyond Ready=True: any pod the -// Deployment considers Available is a candidate for "the current server -// instance". -func podIsReady(p *corev1.Pod) bool { - if p.Status.Phase != corev1.PodRunning { - return false - } - for i := range p.Status.Conditions { - c := &p.Status.Conditions[i] - if c.Type == corev1.PodReady { - return c.Status == corev1.ConditionTrue - } - } - return false -} - -// cascadeRestartEngineDeployments finds every engine pod the webhook -// stamped against this backend, resolves each to its owning Deployment -// via the standard pod→ReplicaSet→Deployment owner chain, and patches -// the trigger annotation onto each unique Deployment's pod template. -// Returns the count of Deployments annotated. -// -// The pod filter is **the injected-by annotation pair**, not the -// EngineSelector. The webhook stamps `inferencecache.io/injected-by` -// AND `inferencecache.io/injected-by-uid` on every successful injection -// — both are required, and the UID half closes the forgery hole that -// `failurePolicy=Ignore` would otherwise leave open (an operator with -// pod-create RBAC could otherwise paste an `injected-by` value -// pointing at any backend and trick the cascade into rolling its -// engines). Filtering on the annotation pair, not the selector, also -// handles the cases the selector-based filter would silently miss: -// - Operator removed `spec.engineSelector` after pods were injected. -// The injected-by stamp persists on the pods, but the selector -// no longer matches them. -// - Pod labels drifted from the selector after admission (a -// redeploy with edited labels). The pod still holds the stale -// LMCache socket, but a label-selector list would miss it. -// -// Why annotate the Deployment's pod template (not the pod): the -// Deployment controller only watches its template; an annotation on a -// child pod has no rolling-restart effect. Patching -// `spec.template.metadata.annotations` is the same mechanism -// `kubectl rollout restart` uses (it stamps -// `kubectl.kubernetes.io/restartedAt`) — we just project-namespace the -// key. -// -// Engine pods owned by non-Deployment workloads (StatefulSet, bare -// Pod, Job, …) are skipped; rolling-restart via `spec.template` -// annotation is a Deployment-shaped contract, and the operator is -// responsible for restarting other workload kinds on a cache-server -// replacement. -// -// The pod List is namespace-scoped (no label selector) because the -// `injected-by` annotation is the authoritative wiring signal and -// annotations cannot be apiserver-side selectors. Namespace-bounded — -// the webhook only stamps `injected-by` on pods in the matched -// backend's namespace. -// -// Trust model: the only authority required to enqueue a cascade- -// restart is the CacheBackendReconciler's own SA, which has the -// `apps/deployments,patch` verb granted via this package's RBAC -// markers. The injected-by + injected-by-uid annotation pair we read -// from pods is normally stamped by the mutating Pod webhook (running -// as the controller SA), so an unprivileged pod-create user cannot -// forge it: when the webhook is reachable it overwrites or strips -// those annotations on every CREATE. The narrow forgery window opens -// only when the webhook is unreachable AT admission time and -// `MutatingWebhookConfiguration.failurePolicy=Ignore` lets the pod -// admit unmodified — in that case a caller who can read live CR / -// ReplicaSet / Deployment UIDs could plant a pod whose annotations + -// ownerRef chain looks legitimate, triggering a cascade-restart of a -// Deployment they do not have direct patch RBAC on. The blast radius -// is bounded to the same namespace as the CacheBackend (pod-list is -// namespace-scoped here, and the webhook only matches CRs in the -// pod's namespace), and a normal cluster keeps the webhook reachable -// — but operators running with hostile namespace tenants should -// either set `failurePolicy=Fail` on the mutating webhook or accept -// that pod-create RBAC in a namespace is elevated to "force-restart -// any in-namespace Deployment whose template the controller can -// patch". The cache plane's existing engine-pod-events controller -// makes the same trust assumption for the same reason. -func (r *CacheBackendReconciler) cascadeRestartEngineDeployments(ctx context.Context, backend *cachev1alpha1.CacheBackend, serverInstanceID string) (int, error) { - reader := client.Reader(r.APIReader) - if reader == nil { - reader = r.Client - } - - var pods corev1.PodList - if err := reader.List(ctx, &pods, client.InNamespace(backend.Namespace)); err != nil { - return 0, fmt.Errorf("list engine pods: %w", err) - } - - wantInjectedBy := backend.Namespace + "/" + backend.Name - wantInjectedByUID := string(backend.UID) - // Dedupe targets by (name, UID) — not name alone. The owner-chain - // walk in podOwningDeploymentName verified the Deployment's UID at - // the moment of resolution, but a Deployment could be deleted and - // re-created under the same name between resolution and the patch - // loop; annotating by name alone in that window would roll an - // unrelated workload that happens to share the name. Carrying the - // expected UID lets annotateDeploymentForCascade re-check before - // patching, closing the TOCTOU window. - type targetRef struct { - uid string - } - targets := map[string]targetRef{} - for i := range pods.Items { - p := &pods.Items[i] - if p.Annotations[enginebinding.AnnotationInjectedBy] != wantInjectedBy { - continue - } - // Require the matching injected-by-uid. The webhook always - // writes both annotations on a successful injection; a pod - // carrying only `injected-by` (or `injected-by-uid` with a - // stale UID) is either a forgery or a survivor from a CR - // that was deleted and recreated. In both cases the pod is - // no longer wired to THIS CR's cache-server, so cascading - // would either roll an unrelated workload or do nothing — - // neither is helpful. - if wantInjectedByUID == "" || p.Annotations[enginebinding.AnnotationInjectedByUID] != wantInjectedByUID { - continue - } - depName, depUID, ok, err := r.podOwningDeployment(ctx, reader, p) - if err != nil { - return 0, err - } - if !ok { - continue - } - // Self-target guard: never cascade-annotate the backend's own - // cache-server Deployment. The canonical owned Deployment is - // named after the backend (see buildDeployment in - // cachebackend_controller.go); podOwningDeployment validates - // UID at every link in the owner chain, so a depName equal to - // backend.Name here means we resolved to OUR Deployment, not - // a foreign Deployment squatting on the same name. - // - // Without this guard, a misconfigured spec.engineSelector that - // overlaps the cache-server pod's labels — combined with a - // webhook decision to stamp the cache-server pod with this - // backend's injected-by + injected-by-uid annotations — would - // pull the cache-server Deployment into the target set. The - // resulting annotate-patch bumps the pod template, the - // Deployment rolls a new cache-server pod, the controller - // observes the new pod's UID, and reconcileServerInstance - // fires another cascade — an infinite self-induced rollout - // loop. The cache-server's own recovery is observation-driven - // (status.observedServerInstance); a forced rollout from - // this path is never the right answer. - if depName == backend.Name { - continue - } - // First write wins; if a later pod resolves the same name to a - // different UID, that means the chain has churned since the - // pod List — keep the UID we saw first (the patch step will - // reject if neither identity matches the live Deployment). - if _, exists := targets[depName]; !exists { - targets[depName] = targetRef{uid: depUID} - } - } - - // Sort for deterministic patch order (helps tests, helps log - // readability; the apiserver does not care). - names := make([]string, 0, len(targets)) - for n := range targets { - names = append(names, n) - } - sort.Strings(names) - - annotated := 0 - for _, name := range names { - patched, err := r.annotateDeploymentForCascade(ctx, backend.Namespace, name, targets[name].uid, serverInstanceID) - if err != nil { - return annotated, err - } - if patched { - annotated++ - } - } - return annotated, nil -} - -// podOwningDeployment walks pod → controller-owning ReplicaSet → -// controller-owning Deployment and returns the Deployment's name AND -// observed UID (in the same namespace as the pod — apps/v1 ownership -// is namespaced). The UID is carried back so the caller can re-verify -// at the moment of patch, closing a TOCTOU window where the resolved -// Deployment is deleted and re-created under the same name between -// resolution and annotate. -// -// The (found, err) split distinguishes "definitively not Deployment- -// owned" (found=false, err=nil — bare pod, StatefulSet, etc.) from -// "couldn't decide" (err != nil — transient apiserver / RBAC issue). -// Callers MUST propagate the error so the cascade doesn't advance -// status.observedServerInstance over an incomplete picture; silently -// collapsing transient errors to "not owned" would let the cascade -// skip a Deployment whose owner chain happened to fail to Get, while -// the latch still moves forward, leaving the engine pods with stale -// sockets that will never be retried. -// -// Each link is checked on both name AND UID: a name-only match would -// resolve a stale or forged ownerRef to the wrong live object (a -// Deployment recreated under the same name is the canonical bad -// case). The (kind, apiVersion) check rejects ownerRefs to non- -// apps/v1 ReplicaSets/Deployments (defensive — a CRD shaped like a -// ReplicaSet/Deployment from another apiGroup must not be picked up). -// -// A genuine NotFound on the ReplicaSet or Deployment (the chain has -// been GCd between our pod List and the Get) returns -// (found=false, err=nil) — the pod's owner chain is gone, so it -// cannot be cascaded anyway. Non-NotFound errors bubble up. -func (r *CacheBackendReconciler) podOwningDeployment(ctx context.Context, reader client.Reader, pod *corev1.Pod) (string, string, bool, error) { - rsRef := metav1.GetControllerOf(pod) - if rsRef == nil || rsRef.Kind != "ReplicaSet" || !ownerRefIsAppsV1(rsRef) { - return "", "", false, nil - } - var rs appsv1.ReplicaSet - if err := reader.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: rsRef.Name}, &rs); err != nil { - if apierrors.IsNotFound(err) { - return "", "", false, nil - } - return "", "", false, fmt.Errorf("get owning ReplicaSet %s/%s: %w", pod.Namespace, rsRef.Name, err) - } - if rs.UID != rsRef.UID { - // Name resolved to a different live RS (re-created under the - // same name). Not the pod's actual owner. - return "", "", false, nil - } - depRef := metav1.GetControllerOf(&rs) - if depRef == nil || depRef.Kind != "Deployment" || !ownerRefIsAppsV1(depRef) { - return "", "", false, nil - } - // Verify the Deployment named in depRef still exists with the - // declared UID. A name-only return would let a stale ownerRef - // resolve to a brand-new Deployment that happens to share the - // name — which we'd then cascade-restart unrelated work. - var dep appsv1.Deployment - if err := reader.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: depRef.Name}, &dep); err != nil { - if apierrors.IsNotFound(err) { - return "", "", false, nil - } - return "", "", false, fmt.Errorf("get owning Deployment %s/%s: %w", pod.Namespace, depRef.Name, err) - } - if dep.UID != depRef.UID { - return "", "", false, nil - } - return dep.Name, string(dep.UID), true, nil -} - -// annotateDeploymentForCascade patches AnnotationCacheServerRestartTrigger -// onto the Deployment's pod template annotations using a JSON merge -// patch, so concurrent writers on other template fields are not -// clobbered. Returns whether a patch was actually issued (false when -// the trigger already carried serverInstanceID — guards against a -// no-op rollout if the reconciler retries on the same identifier). -// -// expectedUID is the Deployment UID observed at owner-chain resolution -// time. The TOCTOU window between resolution and patch is closed at -// TWO points: -// -// 1. The pre-patch Get goes through APIReader (uncached) so the UID -// we compare against is the live apiserver value, not a stale -// cached object. Without this, a cache that hadn't yet seen a -// same-name recreate would pass the UID check against the -// pre-recreate object and we would then patch the post-recreate -// unrelated Deployment. -// -// 2. The Patch carries an optimistic-lock precondition derived from -// the resourceVersion read in step 1 (MergeFromWithOptimisticLock). -// If the live Deployment has been updated OR deleted-and-recreated -// between our Get and the Patch, the apiserver returns Conflict -// and the patch is rejected — even though it addresses by name. -// We surface the conflict as an error so the cascade retries on -// the next reconcile against the freshest view. -// -// A NotFound on the live Deployment (the target has been GCd in the -// resolution-to-patch window) returns (false, nil) — there is -// nothing to roll. A UID mismatch likewise returns (false, nil) — -// the same-name workload that exists today is not ours. -func (r *CacheBackendReconciler) annotateDeploymentForCascade(ctx context.Context, namespace, name, expectedUID, serverInstanceID string) (bool, error) { - // Use the uncached reader for the UID/resourceVersion read; see - // the godoc above. Fall back to the cached client only when - // APIReader is unset (unit-test wiring constructs reconcilers - // without one). - reader := client.Reader(r.APIReader) - if reader == nil { - reader = r.Client - } - var dep appsv1.Deployment - if err := reader.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &dep); err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, fmt.Errorf("get engine deployment %s/%s: %w", namespace, name, err) - } - if string(dep.UID) != expectedUID { - // Live Deployment has a different UID than the one resolved - // during pod owner-chain walking — same name, different - // object. Refuse to roll the unrelated workload. - return false, nil - } - if dep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] == serverInstanceID { - // Already up to date — somebody (us, on a retry) already - // patched this round. Skip without bumping the rollout - // revision. - return false, nil - } - before := dep.DeepCopy() - if dep.Spec.Template.Annotations == nil { - dep.Spec.Template.Annotations = map[string]string{} - } - dep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] = serverInstanceID - // MergeFromWithOptimisticLock embeds before.ResourceVersion as a - // precondition on the merge patch: if the apiserver-side object - // has been updated (including delete+recreate, which assigns a - // fresh resourceVersion) since the Get above, the patch is - // rejected with Conflict. Surface as an error so the next - // reconcile retries against the freshest view. - patch := client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{}) - if err := r.Patch(ctx, &dep, patch); err != nil { - return false, fmt.Errorf("patch engine deployment %s/%s pod-template annotations: %w", namespace, name, err) - } - return true, nil -} diff --git a/internal/controller/cachebackend_server_restart_integration_test.go b/internal/controller/cachebackend_server_restart_integration_test.go deleted file mode 100644 index a8c45a53..00000000 --- a/internal/controller/cachebackend_server_restart_integration_test.go +++ /dev/null @@ -1,432 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "testing" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" -) - -// TestIntegrationCacheBackendServerRestartCascade exercises the -// cache-server restart cascade against a real apiserver (envtest), so -// it covers behavior the fake client can't: real Patch + Status().Patch -// semantics, real Pod readiness-condition handling, and the -// pod→ReplicaSet→Deployment owner-resolution chain Get'd against a -// real apiserver. The reconciler here is constructed without -// APIReader, so all reads go through the embedded client.Client. -// -// What's not covered here: a live kubelet rolling a Pod or restarting -// a container in place. envtest has no kubelet, so we simulate "the -// cache-server pod was replaced" by deleting the old Pod and creating -// a new one with the same selector labels but a fresh UID. In-place -// container-restart detection (the restart-sum half of the server- -// instance identifier) is covered by the fake-client unit suite, -// which can mutate status.containerStatuses without a kubelet. -func TestIntegrationCacheBackendServerRestartCascade(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, _ := startEnv(t) - r := &CacheBackendReconciler{ - Client: k8s, - Scheme: scheme, - Log: logr.Discard(), - MinServerRestartCascadeInterval: 100 * time.Millisecond, - serverInstanceCascade: newServerInstanceCascade(), - } - ctx := context.Background() - - t.Run("UIDTransitionAnnotatesEngineDeployment", func(t *testing.T) { - ns := freshNS(t, k8s) - resetBackendServerRestartCascadesTotalForTest() - - cb := lmcacheBackend("cache", ns) - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm-engine"}, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - // Engine wiring: a Deployment owns a ReplicaSet which owns a - // Pod stamped by the webhook. We build the chain by hand - // (envtest doesn't run the Deployment controller). - engineDep := newEngineDeployment(ns, "vllm-engine") - if err := k8s.Create(ctx, engineDep); err != nil { - t.Fatalf("create engine deployment: %v", err) - } - engineRS := newEngineReplicaSet(ns, "vllm-engine-rs", engineDep) - if err := k8s.Create(ctx, engineRS); err != nil { - t.Fatalf("create engine RS: %v", err) - } - fetchAfterCreate(t, k8s, engineDep) - fetchAfterCreate(t, k8s, engineRS) - enginePod := newEngineInjectedPod(ns, "vllm-engine-aaa", engineRS, ns, "cache", string(cb.UID)) - if err := k8s.Create(ctx, enginePod); err != nil { - t.Fatalf("create engine pod: %v", err) - } - - // First Ready cache-server pod. The reconciler should see - // this and persist it as the baseline (no cascade). The - // cache-server pod must be owner-referenced up to the - // reconciler-created Deployment so currentServerInstanceID's - // transitive-ownership check admits it. The reconciler creates - // the Deployment on the first reconcile; the RS that would - // normally own pods is fabricated here (envtest runs no apps - // controller). - reconcile(t, r, "cache", ns) - serverRS1 := newServerReplicaSet(t, k8s, ns, "cache", "cache-rs-1") - serverPod1 := newReadyServerPod(ns, "cache-pod-1", "cache") - setServerPodOwner(serverPod1, serverRS1) - createReady(t, k8s, serverPod1) - reconcile(t, r, "cache", ns) - - reloaded := getBackend(t, r, "cache", ns) - if got := reloaded.Status.ObservedServerInstance; got != serverInstanceID(serverPod1) { - t.Fatalf("baseline ObservedServerInstance = %q, want %q", got, serverInstanceID(serverPod1)) - } - gotDep := &appsv1.Deployment{} - if err := k8s.Get(ctx, types.NamespacedName{Name: "vllm-engine", Namespace: ns}, gotDep); err != nil { - t.Fatalf("get engine dep: %v", err) - } - if _, ok := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("engine deployment annotated on first observation; want no cascade yet") - } - if got := cascadeRestartsCount(t, ns, "cache", cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("counter = %v, want 0 (no cascade on first observation)", got) - } - - // Simulate the cache-server pod restarting: delete + recreate - // with a fresh UID. The Pod controller in envtest does not - // run; the test is the authority on what pods exist. The - // replacement pod is owner-referenced to the same RS as the - // first — that's what a rolling restart of the Deployment - // would produce. - if err := k8s.Delete(ctx, serverPod1); err != nil { - t.Fatalf("delete first server pod: %v", err) - } - serverPod2 := newReadyServerPod(ns, "cache-pod-2", "cache") - setServerPodOwner(serverPod2, serverRS1) - createReady(t, k8s, serverPod2) - - reconcile(t, r, "cache", ns) - - reloaded = getBackend(t, r, "cache", ns) - if got := reloaded.Status.ObservedServerInstance; got != serverInstanceID(serverPod2) { - t.Fatalf("ObservedServerInstance after UID flip = %q, want %q", got, serverInstanceID(serverPod2)) - } - if err := k8s.Get(ctx, types.NamespacedName{Name: "vllm-engine", Namespace: ns}, gotDep); err != nil { - t.Fatalf("get engine dep: %v", err) - } - annot := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] - if annot != serverInstanceID(serverPod2) { - t.Fatalf("cascade annotation = %q, want %q", annot, serverInstanceID(serverPod2)) - } - if got := cascadeRestartsCount(t, ns, "cache", cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("counter = %v, want 1", got) - } - }) - - t.Run("RateLimitedSecondCascadeIsDeferred", func(t *testing.T) { - ns := freshNS(t, k8s) - resetBackendServerRestartCascadesTotalForTest() - - // Use a long window so the second cascade is definitely - // inside it. Restored after the test so other subtests use - // the snug 100ms. - prev := r.MinServerRestartCascadeInterval - r.MinServerRestartCascadeInterval = 1 * time.Hour - t.Cleanup(func() { r.MinServerRestartCascadeInterval = prev }) - - cb := lmcacheBackend("cache", ns) - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm-engine"}, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - engineDep := newEngineDeployment(ns, "vllm-engine") - if err := k8s.Create(ctx, engineDep); err != nil { - t.Fatalf("create engine deployment: %v", err) - } - engineRS := newEngineReplicaSet(ns, "vllm-engine-rs", engineDep) - if err := k8s.Create(ctx, engineRS); err != nil { - t.Fatalf("create engine RS: %v", err) - } - fetchAfterCreate(t, k8s, engineDep) - fetchAfterCreate(t, k8s, engineRS) - enginePod := newEngineInjectedPod(ns, "vllm-engine-aaa", engineRS, ns, "cache", string(cb.UID)) - if err := k8s.Create(ctx, enginePod); err != nil { - t.Fatalf("create engine pod: %v", err) - } - - // Baseline observation + first cascade. The cache-server pod - // must be owner-referenced up to the reconciler-created - // Deployment so currentServerInstanceID's transitive- - // ownership check admits it. - reconcile(t, r, "cache", ns) - serverRS := newServerReplicaSet(t, k8s, ns, "cache", "cache-rs-1") - serverPod1 := newReadyServerPod(ns, "cache-pod-1", "cache") - setServerPodOwner(serverPod1, serverRS) - createReady(t, k8s, serverPod1) - reconcile(t, r, "cache", ns) - - serverPod2 := newReadyServerPod(ns, "cache-pod-2", "cache") - setServerPodOwner(serverPod2, serverRS) - if err := k8s.Delete(ctx, serverPod1); err != nil { - t.Fatalf("delete first server pod: %v", err) - } - createReady(t, k8s, serverPod2) - reconcile(t, r, "cache", ns) - - gotDep := &appsv1.Deployment{} - if err := k8s.Get(ctx, types.NamespacedName{Name: "vllm-engine", Namespace: ns}, gotDep); err != nil { - t.Fatalf("get engine dep: %v", err) - } - firstCascadeUID := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] - if firstCascadeUID != serverInstanceID(serverPod2) { - t.Fatalf("first cascade annotation = %q, want %q", firstCascadeUID, serverInstanceID(serverPod2)) - } - if got := cascadeRestartsCount(t, ns, "cache", cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("counter after first cascade = %v, want 1", got) - } - - // Second back-to-back UID flip while still inside the 1h - // rate-limit window. The Deployment annotation must stay - // pinned to the first cascade's UID; status must stay pinned - // likewise. - serverPod3 := newReadyServerPod(ns, "cache-pod-3", "cache") - setServerPodOwner(serverPod3, serverRS) - if err := k8s.Delete(ctx, serverPod2); err != nil { - t.Fatalf("delete second server pod: %v", err) - } - createReady(t, k8s, serverPod3) - reconcile(t, r, "cache", ns) - - if err := k8s.Get(ctx, types.NamespacedName{Name: "vllm-engine", Namespace: ns}, gotDep); err != nil { - t.Fatalf("get engine dep: %v", err) - } - if got := gotDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != firstCascadeUID { - t.Fatalf("cascade annotation drifted while rate-limited: got %q, want pinned %q", got, firstCascadeUID) - } - reloaded := getBackend(t, r, "cache", ns) - if got := reloaded.Status.ObservedServerInstance; got != firstCascadeUID { - t.Fatalf("ObservedServerInstance drifted while rate-limited: got %q, want pinned %q", got, firstCascadeUID) - } - if got := cascadeRestartsCount(t, ns, "cache", cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("counter after rate-limited cascade = %v, want 1", got) - } - }) -} - -// newEngineDeployment fabricates a minimal apps/v1 Deployment shaped like a -// vLLM engine workload — enough for the cascade tests' selector match + -// owner-resolution + Status().Patch. -func newEngineDeployment(namespace, name string) *appsv1.Deployment { - one := int32(1) - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "vllm-engine"}}, - Spec: appsv1.DeploymentSpec{ - Replicas: &one, - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vllm-engine"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "vllm-engine"}}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "vllm", Image: "vllm:test"}}, - }, - }, - }, - } -} - -// newEngineReplicaSet fabricates the ReplicaSet the apps/v1 Deployment -// controller would normally create (envtest does not run that -// controller). The Deployment is named via a controller-owner reference so -// podOwningDeployment can walk pod → RS → Deployment. -func newEngineReplicaSet(namespace, name string, dep *appsv1.Deployment) *appsv1.ReplicaSet { - tru := true - one := int32(1) - return &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{"app": "vllm-engine"}, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: dep.Name, - UID: dep.UID, - Controller: &tru, - }}, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &one, - Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vllm-engine"}}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "vllm-engine"}}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "vllm", Image: "vllm:test"}}, - }, - }, - }, - } -} - -// newEngineInjectedPod fabricates an engine pod that already carries the -// pod-webhook's injected-by annotations, so the cascade filter (which -// gates on the annotation, NOT just the selector) admits it. -func newEngineInjectedPod(namespace, name string, rs *appsv1.ReplicaSet, backendNS, backendName, backendUID string) *corev1.Pod { - tru := true - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{"app": "vllm-engine"}, - Annotations: map[string]string{ - podwebhook.AnnotationInjectedBy: backendNS + "/" + backendName, - podwebhook.AnnotationInjectedByUID: backendUID, - }, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: rs.Name, - UID: rs.UID, - Controller: &tru, - }}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "vllm", Image: "vllm:test"}}, - }, - } -} - -// newReadyServerPod builds a cache-server pod labeled the way the -// reconciler labels its own children. UID is assigned by envtest on -// create; the caller reads it back to compare against -// status.observedServerInstance. -func newReadyServerPod(namespace, name, backendName string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: map[string]string{ - "app.kubernetes.io/name": "cachebackend", - "app.kubernetes.io/instance": backendName, - "app.kubernetes.io/managed-by": "inference-cache-controller", - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "lmcache-server", Image: "lmcache:test"}}, - }, - } -} - -// createReady creates the pod and then patches its status to Running + -// Ready=True (envtest does not run kubelet, so spec.status is otherwise -// empty). The caller can read pod.UID after this returns. -func createReady(t *testing.T, k8s interface { - Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error - Status() client.StatusWriter - Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error -}, pod *corev1.Pod) { - t.Helper() - ctx := context.Background() - if err := k8s.Create(ctx, pod); err != nil { - t.Fatalf("create server pod %s: %v", pod.Name, err) - } - // Status().Patch with a snapshot: refetch the freshly-created pod so - // the patch base carries the apiserver-assigned ResourceVersion. - live := &corev1.Pod{} - if err := k8s.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, live); err != nil { - t.Fatalf("refetch server pod %s: %v", pod.Name, err) - } - live.Status.Phase = corev1.PodRunning - live.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} - if err := k8s.Status().Update(ctx, live); err != nil { - t.Fatalf("set server pod %s ready: %v", pod.Name, err) - } - // Copy UID back so the caller can compare. - pod.UID = live.UID -} - -// fetchAfterCreate refreshes obj's ResourceVersion + UID after a Create, -// so chained creates that reference obj.UID see the value the apiserver -// assigned. -func fetchAfterCreate(t *testing.T, k8s interface { - Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error -}, obj client.Object) { - t.Helper() - if err := k8s.Get(context.Background(), client.ObjectKeyFromObject(obj), obj); err != nil { - t.Fatalf("refetch after create: %v", err) - } -} - -// newServerReplicaSet fabricates the ReplicaSet the apps/v1 Deployment -// controller would normally create for the reconciler-managed cache- -// server Deployment. envtest runs no apps controller, so the test is -// the authority on what ReplicaSets/Pods exist. The RS is owner- -// referenced to the reconciler-created Deployment (looked up after the -// first reconcile creates it) so currentServerInstanceID's transitive -// ownership check (pod → RS → Deployment) admits owned pods. -func newServerReplicaSet(t *testing.T, k8s client.Client, namespace, backendName, rsName string) *appsv1.ReplicaSet { - t.Helper() - dep := &appsv1.Deployment{} - if err := k8s.Get(context.Background(), client.ObjectKey{Namespace: namespace, Name: backendName}, dep); err != nil { - t.Fatalf("server Deployment %s/%s not present (run reconcile first): %v", namespace, backendName, err) - } - tru := true - one := int32(1) - rs := &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: rsName, - Namespace: namespace, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: dep.Name, - UID: dep.UID, - Controller: &tru, - }}, - }, - Spec: appsv1.ReplicaSetSpec{ - Replicas: &one, - Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(backendName)}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: selectorLabels(backendName)}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "lmcache-server", Image: "lmcache:test"}}, - }, - }, - }, - } - if err := k8s.Create(context.Background(), rs); err != nil { - t.Fatalf("create server RS: %v", err) - } - fetchAfterCreate(t, k8s, rs) - return rs -} - -// setServerPodOwner stamps the controller-owner reference from a -// cache-server pod up to its RS — the missing link the test needs to -// build for envtest (the apps controller would normally do this). -func setServerPodOwner(pod *corev1.Pod, rs *appsv1.ReplicaSet) { - tru := true - pod.OwnerReferences = append(pod.OwnerReferences, metav1.OwnerReference{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: rs.Name, - UID: rs.UID, - Controller: &tru, - }) -} diff --git a/internal/controller/cachebackend_server_restart_test.go b/internal/controller/cachebackend_server_restart_test.go deleted file mode 100644 index 7d9617a6..00000000 --- a/internal/controller/cachebackend_server_restart_test.go +++ /dev/null @@ -1,1873 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/go-logr/logr" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" -) - -// cascadeRestartFixture builds a fake-client-backed reconciler plus a fully -// wired managed CacheBackend with one Ready cache-server pod (transitively -// owned by the CacheBackend-owned Deployment+ReplicaSet, the way the apps -// controller stack would create them) and one engine -// Deployment+ReplicaSet+Pod that the webhook has injected against the -// backend. Shared by every cascade test so each scenario only expresses -// what's different (UID, status, rate-limit window, …) and the boring -// setup stays terse. -type cascadeRestartFixture struct { - r *CacheBackendReconciler - backend *cachev1alpha1.CacheBackend - serverDep *appsv1.Deployment - serverRS *appsv1.ReplicaSet - serverPod *corev1.Pod - engineDep *appsv1.Deployment - engineRS *appsv1.ReplicaSet - enginePod *corev1.Pod - engineNS string - cacheNS string - cacheName string - engineDepN string - enginePodN string -} - -func newCascadeRestartFixture(t *testing.T, opts ...func(*cascadeRestartFixture)) *cascadeRestartFixture { - t.Helper() - f := &cascadeRestartFixture{ - cacheNS: "team-a", - cacheName: "cache", - engineNS: "team-a", - engineDepN: "vllm-engine", - enginePodN: "vllm-engine-abc", - } - for _, o := range opts { - o(f) - } - - scheme := newScheme(t) - - f.backend = &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.cacheName, - Namespace: f.cacheNS, - UID: "cache-uid-1", - Generation: 1, - }, - Spec: cachev1alpha1.CacheBackendSpec{ - Type: cachev1alpha1.CacheBackendTypeLMCache, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm-engine"}, - }, - }, - } - - tru := true - // Cache-server Deployment+ReplicaSet the reconciler "owns" — the - // transitive owner chain currentServerInstanceID's strengthened - // ownership check requires to attribute a Ready pod to this backend. - // The Deployment's controller-owner reference points at the - // CacheBackend, which is what IsControlledBy expects. - f.serverDep = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.cacheName, - Namespace: f.cacheNS, - UID: "cache-dep-uid", - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: cachev1alpha1.GroupVersion.String(), - Kind: "CacheBackend", - Name: f.backend.Name, - UID: f.backend.UID, - Controller: &tru, - }}, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: selectorLabels(f.cacheName)}, - // containerRunSum scopes its restart-count sum to - // container names from THIS template, so the test - // must enumerate the cache-server's container name - // (lmcache-server) — sidecars added to the pod by - // other admission webhooks would not be included. - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "lmcache-server", Image: "lmcache:test"}}, - }, - }, - }, - } - f.serverRS = &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.cacheName + "-rs", - Namespace: f.cacheNS, - UID: "cache-rs-uid", - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: f.cacheName, - UID: f.serverDep.UID, - Controller: &tru, - }}, - }, - } - // The "current Ready" cache-server pod the controller observes. - // Labeled with the exact selectorLabels() set and owner-referenced - // up the chain to serverDep so currentServerInstanceID's transitive - // ownership check admits it. - f.serverPod = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-aaa", - Namespace: f.cacheNS, - UID: "cache-pod-uid-1", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{ - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }}, - }, - } - - // Engine Deployment, the cascade target. - f.engineDep = &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.engineDepN, - Namespace: f.engineNS, - UID: "engine-dep-uid", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"app": "vllm-engine"}, - }, - }, - }, - } - f.engineRS = &appsv1.ReplicaSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.engineDepN + "-rs", - Namespace: f.engineNS, - UID: "engine-rs-uid", - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: f.engineDepN, - UID: f.engineDep.UID, - Controller: &tru, - }}, - }, - } - f.enginePod = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: f.enginePodN, - Namespace: f.engineNS, - UID: "engine-pod-uid", - Labels: map[string]string{"app": "vllm-engine"}, - Annotations: map[string]string{ - podwebhook.AnnotationInjectedBy: f.cacheNS + "/" + f.cacheName, - podwebhook.AnnotationInjectedByUID: string(f.backend.UID), - }, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.engineRS.Name, - UID: f.engineRS.UID, - Controller: &tru, - }}, - }, - } - - c := fake.NewClientBuilder(). - WithScheme(scheme). - WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &appsv1.Deployment{}). - WithObjects(f.backend, f.serverDep, f.serverRS, f.serverPod, f.engineDep, f.engineRS, f.enginePod). - Build() - f.r = &CacheBackendReconciler{ - Client: c, - Scheme: scheme, - Log: logr.Discard(), - MinServerRestartCascadeInterval: 50 * time.Millisecond, // tests run with a tiny window - serverInstanceCascade: newServerInstanceCascade(), - } - - resetBackendServerRestartCascadesTotalForTest() - return f -} - -func (f *cascadeRestartFixture) reload(t *testing.T) { - t.Helper() - cb := &cachev1alpha1.CacheBackend{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.cacheName, Namespace: f.cacheNS}, cb); err != nil { - t.Fatalf("reload backend: %v", err) - } - f.backend = cb -} - -func (f *cascadeRestartFixture) reloadEngineDep(t *testing.T) { - t.Helper() - dep := &appsv1.Deployment{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.engineDepN, Namespace: f.engineNS}, dep); err != nil { - t.Fatalf("reload engine dep: %v", err) - } - f.engineDep = dep -} - -// serverInstanceID is the per-pod identifier currentServerInstanceID -// computes: :. Shared by the assertion -// helpers so tests build the expected observedServerInstance value -// without duplicating the format. Mirrors containerRunSum in -// cachebackend_server_restart.go. -func serverInstanceID(p *corev1.Pod) string { - var sum int32 - for i := range p.Status.ContainerStatuses { - sum += p.Status.ContainerStatuses[i].RestartCount - } - return fmt.Sprintf("%s:%d", p.UID, sum) -} - -func cascadeRestartsCount(t *testing.T, namespace, backend, reason string) float64 { - t.Helper() - m, err := backendServerRestartCascadesTotal.GetMetricWithLabelValues(namespace, backend, reason) - if err != nil { - t.Fatalf("get counter %s/%s/%s: %v", namespace, backend, reason, err) - } - var pb dto.Metric - if err := m.(prometheus.Counter).Write(&pb); err != nil { - t.Fatalf("write counter: %v", err) - } - return pb.GetCounter().GetValue() -} - -func TestReconcileServerInstance_FirstObservationStampsBaseline(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Status starts empty (no ObservedServerInstance). The first call - // should persist the UID baseline and NOT cascade-restart. - if got := f.backend.Status.ObservedServerInstance; got != "" { - t.Fatalf("precondition: ObservedServerInstance = %q, want empty", got) - } - - wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend) - if wait != 0 { - t.Fatalf("wait = %v, want 0 (first observation never rate-limits)", wait) - } - - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != serverInstanceID(f.serverPod) { - t.Fatalf("ObservedServerInstance = %q, want %q", got, serverInstanceID(f.serverPod)) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("engine deployment got cascade annotation on first observation; want no cascade until a UID transition") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter = %v, want 0 (first observation never cascades)", got) - } -} - -func TestReconcileServerInstance_UIDChangeCascadesEngineDeployment(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Seed status with the prior UID so the call observes a transition. - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend) - if wait != 0 { - t.Fatalf("wait = %v, want 0 (rate-limit window has not been used before)", wait) - } - - f.reloadEngineDep(t) - got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] - if got != serverInstanceID(f.serverPod) { - t.Fatalf("cascade annotation = %q, want %q (the new cache-server pod UID)", got, serverInstanceID(f.serverPod)) - } - - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != serverInstanceID(f.serverPod) { - t.Fatalf("ObservedServerInstance = %q, want %q", got, serverInstanceID(f.serverPod)) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter = %v, want 1", got) - } -} - -func TestReconcileServerInstance_RateLimitedSecondCascadeIsDeferred(t *testing.T) { - f := newCascadeRestartFixture(t) - f.r.MinServerRestartCascadeInterval = 1 * time.Hour // make the window effectively block any second cascade - - // Seed prior UID and a fresh ready pod with the current UID; first call cascades. - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("first cascade wait = %v, want 0", wait) - } - - // Now simulate a second UID flip: replace the server pod with a fresh one carrying a new UID. - if err := f.r.Delete(context.Background(), f.serverPod); err != nil { - t.Fatalf("delete first server pod: %v", err) - } - newPod := f.serverPod.DeepCopy() - newPod.ResourceVersion = "" - newPod.Name = "cache-pod-bbb" - newPod.UID = "cache-pod-uid-2" - if err := f.r.Create(context.Background(), newPod); err != nil { - t.Fatalf("create second server pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), newPod); err != nil { - t.Fatalf("update second server pod status: %v", err) - } - - f.reload(t) - wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend) - if wait <= 0 { - t.Fatalf("wait = %v, want > 0 (rate-limit must defer the second cascade)", wait) - } - if wait > f.r.MinServerRestartCascadeInterval { - t.Fatalf("wait = %v, want <= window %v", wait, f.r.MinServerRestartCascadeInterval) - } - - f.reload(t) - // Status MUST stay pinned to the first cascade's UID — advancing it - // inside the rate-limit window would lose the missed cascade. - if got := f.backend.Status.ObservedServerInstance; got != serverInstanceID(f.serverPod) { - t.Fatalf("ObservedServerInstance = %q, want pinned to first-cascade UID %q", got, serverInstanceID(f.serverPod)) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter = %v, want 1 (rate-limited second cascade must not increment)", got) - } -} - -func TestReconcileServerInstance_NotReadyPodGivesNoBaseline(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Flip the server pod to NOT Ready (Pending), simulating mid-rollout. - notReady := f.serverPod.DeepCopy() - notReady.Status.Phase = corev1.PodPending - notReady.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionFalse}} - if err := f.r.Status().Update(context.Background(), notReady); err != nil { - t.Fatalf("flip pod to not-ready: %v", err) - } - - wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend) - if wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != "" { - t.Fatalf("ObservedServerInstance = %q, want empty (no Ready pod to anchor to)", got) - } -} - -// TestReconcileServerInstance_SelectorRemovedButPodStillInjectedCascades -// asserts that an operator clearing spec.engineSelector AFTER engine -// pods were already injected does not silently break recovery. The -// pods' injected-by annotations persist, their LMCache sockets are -// still stale on a cache-server restart, and the cascade MUST still -// roll them. Selector match is an apiserver-side perf optimization -// for other reconciler paths; the cascade authoritatively filters on -// the injected-by annotation pair, so removing the selector does not -// disable recovery. -func TestReconcileServerInstance_SelectorRemovedButPodStillInjectedCascades(t *testing.T) { - f := newCascadeRestartFixture(t) - f.backend.Spec.EngineSelector = nil - if err := f.r.Update(context.Background(), f.backend); err != nil { - t.Fatalf("update backend: %v", err) - } - f.reload(t) - - // Seed prior UID so this is a transition, not the first observation. - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - - f.reloadEngineDep(t) - got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] - if got != serverInstanceID(f.serverPod) { - t.Fatalf("cascade annotation = %q, want %q (already-injected pods must still cascade after selector removal)", got, serverInstanceID(f.serverPod)) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != serverInstanceID(f.serverPod) { - t.Fatalf("ObservedServerInstance = %q, want %q", got, serverInstanceID(f.serverPod)) - } -} - -// TestReconcileServerInstance_StaleInjectedByUIDIsRejected asserts -// that the cascade rejects a pod whose injected-by name matches but -// whose injected-by-uid does not (CR deleted and recreated under the -// same name, or an operator with pod-create RBAC forging the -// annotation). The pod is NOT actually wired to the live CR's cache- -// server socket — annotating its Deployment would roll unrelated -// work or do nothing useful. -func TestReconcileServerInstance_StaleInjectedByUIDIsRejected(t *testing.T) { - f := newCascadeRestartFixture(t) - // Forge a name-match / UID-mismatch on the engine pod. - enginePod := f.enginePod.DeepCopy() - enginePod.Annotations[podwebhook.AnnotationInjectedByUID] = "stale-uid-from-deleted-cr" - if err := f.r.Update(context.Background(), enginePod); err != nil { - t.Fatalf("stale UID annotation: %v", err) - } - - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("engine deployment cascaded against a pod with a stale injected-by-uid; want no cascade") - } -} - -// TestReconcileServerInstance_ForeignReadyPodIgnoredForServerInstance -// asserts that a Ready pod carrying the controller-managed labels but -// NOT controller-owned by THIS backend's Deployment must not advance -// observedServerInstance — otherwise a transition would spuriously -// trigger an engine rollout. -func TestReconcileServerInstance_ForeignReadyPodIgnoredForServerInstance(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Pre-seed status to the existing real pod's UID so the next - // observation is a no-op transition rather than first-observation. - f.backend.Status.ObservedServerInstance = serverInstanceID(f.serverPod) - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - // Foreign pod: same labels, NOT owned via the cache-server - // Deployment chain. A name lex-smaller than the legit cache pod - // so a label-only picker would prefer it. - foreign := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "aaaa-foreign-pod", - Namespace: f.cacheNS, - UID: "foreign-uid", - Labels: selectorLabels(f.cacheName), - // No ownerRefs — looks like a bare pod from another tool. - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{ - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }}, - }, - } - if err := f.r.Create(context.Background(), foreign); err != nil { - t.Fatalf("create foreign pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), foreign); err != nil { - t.Fatalf("set foreign ready: %v", err) - } - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != serverInstanceID(f.serverPod) { - t.Fatalf("ObservedServerInstance = %q, want pinned to the legit pod %q (foreign pod must not advance the latch)", got, serverInstanceID(f.serverPod)) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("foreign pod triggered a cascade; want no cascade") - } -} - -// TestReconcileServerInstance_MultiReplicaTracksEveryReadyPod asserts -// that a backend with multiple Ready cache-server pods (the ephemeral -// `spec.replicas > 1` shape) encodes every Ready pod's UID into -// observedServerInstance. Replacing ANY one of the replicas must -// advance the identifier and cascade — a tracker that watched only -// one pod would silently miss restarts of the others. -func TestReconcileServerInstance_MultiReplicaTracksEveryReadyPod(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Add a second Ready cache-server pod owned by the same RS. - tru := true - pod2 := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-bbb", - Namespace: f.cacheNS, - UID: "cache-pod-uid-2", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), pod2); err != nil { - t.Fatalf("create pod2: %v", err) - } - if err := f.r.Status().Update(context.Background(), pod2); err != nil { - t.Fatalf("set pod2 ready: %v", err) - } - - // First observation should encode BOTH pods, sorted by name. With - // pod-aaa < pod-bbb, the lex-sorted order is uid-1, uid-2. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("first observation wait = %v, want 0", wait) - } - f.reload(t) - wantInitial := serverInstanceID(f.serverPod) + "," + serverInstanceID(pod2) - if got := f.backend.Status.ObservedServerInstance; got != wantInitial { - t.Fatalf("initial ObservedServerInstance = %q, want %q (both Ready pod UIDs, lex-sorted by name)", got, wantInitial) - } - - // Replace ONLY the second replica (the one whose UID would be - // silently missed by a single-pod tracker). - if err := f.r.Delete(context.Background(), pod2); err != nil { - t.Fatalf("delete pod2: %v", err) - } - pod2b := pod2.DeepCopy() - pod2b.ResourceVersion = "" - pod2b.UID = "cache-pod-uid-2-replacement" - if err := f.r.Create(context.Background(), pod2b); err != nil { - t.Fatalf("create pod2 replacement: %v", err) - } - if err := f.r.Status().Update(context.Background(), pod2b); err != nil { - t.Fatalf("ready pod2 replacement: %v", err) - } - - // The identifier must now advance to include the replacement UID. - // Wait a frame for the rate-limit window — fixture's - // MinServerRestartCascadeInterval is 50ms. - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("replacement wait = %v, want 0", wait) - } - f.reload(t) - wantAfter := serverInstanceID(f.serverPod) + "," + serverInstanceID(pod2b) - if got := f.backend.Status.ObservedServerInstance; got != wantAfter { - t.Fatalf("ObservedServerInstance after replacement = %q, want %q", got, wantAfter) - } - f.reloadEngineDep(t) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != wantAfter { - t.Fatalf("cascade annotation = %q, want %q (non-first replica's restart must still cascade)", got, wantAfter) - } -} - -// TestReconcileServerInstance_RollingUpdateSupersetDoesNotCascade -// asserts that a Deployment rolling-update midpoint — when the old -// pod is still Ready while the new one comes up (maxSurge=1) — -// does NOT trigger a cascade AND does NOT advance -// observedServerInstance. The cascade fires on the NEXT transition -// that drops the old pod, and the latch stays pinned at the prior -// baseline through the midpoint so a rollback (see -// TestReconcileServerInstance_RollingUpdateRollbackDoesNotCascade) -// is a true no-op. Without this debounce a normal single-replica -// rollout would roll the engine fleet twice. -func TestReconcileServerInstance_RollingUpdateSupersetDoesNotCascade(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline: only one pod, observedServerInstance gets stamped. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Simulate the rolling-update midpoint: add a second Ready pod - // owned by the same RS (a new replica from maxSurge=1). - tru := true - newer := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-zzz", // sorts AFTER the original - Namespace: f.cacheNS, - UID: "cache-pod-uid-new", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), newer); err != nil { - t.Fatalf("create newer pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), newer); err != nil { - t.Fatalf("ready newer pod: %v", err) - } - - // Mid-rollout transition: prior strictly grows. Must NOT cascade - // AND must NOT advance the latch — keeping prior pinned is what - // makes a subsequent rollback a no-op. - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("midpoint wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("midpoint ObservedServerInstance = %q, want %q (must stay pinned to baseline through strict-superset; advancing here would make a rollback look like a replacement)", got, baseline) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("rolling-update midpoint triggered a cascade; the old pod is still Ready so the cascade must wait") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter at midpoint = %v, want 0", got) - } - - // Drop the old pod (rolling update finished). NOW the cascade - // must fire — the new pod is what serves traffic, the old pod's - // LMCache sockets are unreachable. - if err := f.r.Delete(context.Background(), f.serverPod); err != nil { - t.Fatalf("delete old pod: %v", err) - } - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-rollout wait = %v, want 0", wait) - } - f.reload(t) - wantFinal := serverInstanceID(newer) - if got := f.backend.Status.ObservedServerInstance; got != wantFinal { - t.Fatalf("final ObservedServerInstance = %q, want %q", got, wantFinal) - } - f.reloadEngineDep(t) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != wantFinal { - t.Fatalf("cascade annotation = %q, want %q (cascade fires once, on the drop of the old pod)", got, wantFinal) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter = %v, want exactly 1 (one rolling update = one cascade)", got) - } -} - -// TestReconcileServerInstance_RollingUpdateRollbackDoesNotCascade -// drives the rollback-of-a-rolling-update scenario: the NEW pod -// becomes Ready briefly (strict-superset midpoint) and is then -// rolled back (new pod killed by failing readiness, leaving the -// ORIGINAL pod alone). This must NOT cascade — the original cache- -// server process and its sockets never changed. The contract that -// makes this work is "do not persist the strict-superset midpoint -// while the Deployment is rolling"; the rollback path then becomes -// a true no-op (prior=current after the rollback completes). -func TestReconcileServerInstance_RollingUpdateRollbackDoesNotCascade(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline observation: latch = original pod's identifier. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Simulate the rolling-update midpoint: a second Ready pod appears - // (maxSurge=1). Strict superset → no cascade AND no latch advance. - tru := true - newer := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-zzz", // sorts AFTER the original - Namespace: f.cacheNS, - UID: "cache-pod-uid-newer-but-doomed", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), newer); err != nil { - t.Fatalf("create newer pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), newer); err != nil { - t.Fatalf("ready newer pod: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("midpoint wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("midpoint ObservedServerInstance = %q, want %q (must stay pinned to baseline; advancing here makes the rollback look like a replacement)", got, baseline) - } - - // Now the rollback: the new pod fails readiness / image-pull / etc. - // and is killed, leaving ONLY the original pod alone. Pre-fix this - // looked like "the new pod was replaced" (prior contained newer, - // current does not) and false-cascaded. Post-fix, since the latch - // never advanced past baseline, the rollback is prior=current, - // no-op. - if err := f.r.Delete(context.Background(), newer); err != nil { - t.Fatalf("delete rolled-back newer pod: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-rollback wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("post-rollback ObservedServerInstance = %q, want %q (rolled-back rolling update must be a no-op; original pod and its sockets never changed)", got, baseline) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("rolled-back rolling update triggered a cascade; the cache-server process never changed, every engine still holds a live socket") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter = %v, want 0 (no real cache-server replacement happened)", got) - } -} - -// TestReconcileServerInstance_ConvergedScaleUpPersistsBaseline drives -// the operator-scale-up scenario: a strict-superset transition where -// the owning Deployment has reached steady state at the wider count -// is a legitimate scale-up, NOT a rolling-update midpoint. The latch -// must advance to include the added pod(s) so a later replacement of -// any of the added pods cascades correctly. -// -// The regression this guards against: if the reconciler unconditionally -// refused to persist superset midpoints, a converged scale-up would -// leave the new pod outside the latch forever; a subsequent OOM-kill -// of just the added pod would be observable to engines (their sockets -// to that pod would die) but not to the controller (prior strictly -// containing current's UIDs at same restart-sums → -// instanceChangeRequiresCascade returns false). The engines would -// be stranded on stale sockets with no recovery path. -func TestReconcileServerInstance_ConvergedScaleUpPersistsBaseline(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline: 1 Ready pod. Mark Deployment converged at replicas=1 - // (the fixture defaults to nil-replicas → 1). - if err := setOwnedDeploymentConverged(f, 1); err != nil { - t.Fatalf("set baseline Deployment converged: %v", err) - } - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Operator scales the backend up to 2 replicas. A second Ready pod - // arrives, owned by the same RS chain. Mark Deployment converged - // at replicas=2 (replicas==readyReplicas==updatedReplicas, with - // observedGeneration current). - tru := true - added := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-zzz", - Namespace: f.cacheNS, - UID: "cache-pod-uid-added", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), added); err != nil { - t.Fatalf("create added pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), added); err != nil { - t.Fatalf("ready added pod: %v", err) - } - if err := setOwnedDeploymentConverged(f, 2); err != nil { - t.Fatalf("set converged Deployment at replicas=2: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-scale-up wait = %v, want 0", wait) - } - f.reload(t) - wantBaseline := serverInstanceID(f.serverPod) + "," + serverInstanceID(added) - if got := f.backend.Status.ObservedServerInstance; got != wantBaseline { - t.Fatalf("post-scale-up ObservedServerInstance = %q, want %q (Deployment converged at the wider count → latch must advance so a later replacement of the added pod cascades)", got, wantBaseline) - } - // No cascade should have fired — adding a pod is not a replacement. - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("scale-up triggered a cascade; only replacements/restarts should cascade") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter after scale-up = %v, want 0", got) - } - - // Now replace ONLY the added pod (different UID). Without the - // converged-superset persist, this would stay a strict superset - // of the original baseline and miss the cascade. With the - // persist, the baseline includes the added pod's UID, so its - // disappearance is a real replacement and the cascade fires. - if err := f.r.Delete(context.Background(), added); err != nil { - t.Fatalf("delete added pod (replacement step 1): %v", err) - } - replacement := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-zzz", // same name, different UID - Namespace: f.cacheNS, - UID: "cache-pod-uid-replacement", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), replacement); err != nil { - t.Fatalf("create replacement pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), replacement); err != nil { - t.Fatalf("ready replacement pod: %v", err) - } - // Stay converged at replicas=2 throughout (the rate-limit + the - // rolling-update test's pattern of 60ms sleep applies). - if err := setOwnedDeploymentConverged(f, 2); err != nil { - t.Fatalf("re-affirm converged Deployment: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-replacement wait = %v, want 0", wait) - } - f.reload(t) - wantFinal := serverInstanceID(f.serverPod) + "," + serverInstanceID(replacement) - if got := f.backend.Status.ObservedServerInstance; got != wantFinal { - t.Fatalf("post-replacement ObservedServerInstance = %q, want %q", got, wantFinal) - } - f.reloadEngineDep(t) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != wantFinal { - t.Fatalf("cascade annotation = %q, want %q (replacement of an added scale-up pod must cascade — without the converged-superset persist, the missing-UID transition would still look like a strict superset and miss the cascade)", got, wantFinal) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter after replacement = %v, want exactly 1", got) - } -} - -// TestReconcileServerInstance_ClearedSentinelOverridesStaleStatus -// drives the failure window in the managed→External→managed -// transition: the in-memory clear ran (shadow gone, cleared -// sentinel set) but the on-cluster status patch FAILED, so the -// status field still holds the prior managed period's identifier. -// Without the sentinel, the next managed-period reconcile would -// read prior = statusField (stale) and misclassify the first new -// Ready pod as a replacement → false-cascade. The sentinel forces -// effectivePrior = "" so the new period starts with a clean -// empty→set baseline. -func TestReconcileServerInstance_ClearedSentinelOverridesStaleStatus(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Plant a stale prior-period status value directly on the CR — - // what a managed→External patch-failure window would leave - // behind. The actual current pod is f.serverPod with a - // different identifier; pre-fix the reconciler would see - // stale != current and cascade. - f.backend.Status.ObservedServerInstance = "stale-prior-period:0" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("plant stale status: %v", err) - } - f.reload(t) - - // Simulate the in-memory side of the failed-clear scenario: - // External path's clearServerInstanceLatchShadow ran (which - // sets the cleared sentinel), the status patch errored, then - // the operator flipped back to managed before the retry. The - // in-memory state for the cascade key is: - // shadow: empty - // cleared: true - // lastAt: empty - key := cascadeKey{namespace: f.backend.Namespace, name: f.backend.Name, uid: string(f.backend.UID)} - f.r.serverInstanceCascade.clear(key) - if !f.r.serverInstanceCascade.isCleared(key) { - t.Fatalf("precondition: cleared sentinel not set after clear()") - } - - // Reconcile in the new managed period. currentID = - // serverInstanceID(f.serverPod) != "stale-prior-period:0". - // With the sentinel: effectivePrior="" → empty→set → no - // cascade, persist new baseline. - // Without the sentinel (the bug): effectivePrior=stale → - // real-change branch → cascade fires. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - - f.reload(t) - want := serverInstanceID(f.serverPod) - if got := f.backend.Status.ObservedServerInstance; got != want { - t.Fatalf("status.observedServerInstance = %q, want %q (sentinel should have driven a clean empty→set persist over the stale value)", got, want) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("cascade fired despite the cleared sentinel; this is the false-cascade scenario the sentinel must prevent (managed→External patch-fail + flip-back-to-managed)") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter = %v, want 0", got) - } - - // recordAttempt happened above (during the empty→set persist), - // which should have cleared the sentinel — verify so the next - // real change DOES cascade correctly. - if f.r.serverInstanceCascade.isCleared(key) { - t.Fatalf("cleared sentinel still set after a successful empty→set baseline persist; recordAttempt must clear it so subsequent changes are detected normally") - } -} - -// TestReconcileServerInstance_ShadowWinsOverStaleStatusOnScaleUpPersistFailure -// drives the scenario where a converged scale-up's baseline persist -// fails: the shadow records the widened pod set ("A:0,B:0") but the -// K8s-resident status field still holds the pre-scale baseline -// ("A:0") because the patch never landed. If the prior were taken -// from the status field (stale) instead of the shadow (current), -// a subsequent replacement of just the added pod ("A:0,B:0" → -// "A:0,C:0") would look like a strict superset of "A:0" and miss -// the cascade — engines that connected to B would be stranded on -// stale sockets forever. -func TestReconcileServerInstance_ShadowWinsOverStaleStatusOnScaleUpPersistFailure(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Step 1: baseline (1 Ready pod, converged, persists cleanly). - if err := setOwnedDeploymentConverged(f, 1); err != nil { - t.Fatalf("set baseline Deployment converged: %v", err) - } - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Step 2: scale up to replicas=2. New pod becomes Ready. Make the - // converged-superset baseline persist FAIL. The shadow should - // record "A:0,B:0" while status stays at "A:0". - tru := true - added := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-added", - Namespace: f.cacheNS, - UID: "cache-pod-uid-added", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), added); err != nil { - t.Fatalf("create added pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), added); err != nil { - t.Fatalf("ready added pod: %v", err) - } - if err := setOwnedDeploymentConverged(f, 2); err != nil { - t.Fatalf("set converged at replicas=2: %v", err) - } - - failOnce := &statusPatchFailingClient{Client: f.r.Client, remaining: 1} - f.r.Client = failOnce - time.Sleep(60 * time.Millisecond) - // Scale-up reconcile: status patch fails. Shadow should advance. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait <= 0 { - t.Fatalf("scale-up reconcile wait = %v, want positive (persist failed)", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("scale-up status field = %q after FAILED persist, want still %q", got, baseline) - } - wantShadow := serverInstanceID(f.serverPod) + "," + serverInstanceID(added) - key := cascadeKey{namespace: f.backend.Namespace, name: f.backend.Name, uid: string(f.backend.UID)} - if got := f.r.serverInstanceCascade.lastAttempt(key); got != wantShadow { - t.Fatalf("shadow after scale-up = %q, want %q (shadow must record the widened baseline so a subsequent replacement is detectable)", got, wantShadow) - } - - // Step 3: replace the added pod (B → C, e.g. OOM-kill of B). - // The shadow holds "A:0,B:0"; if the reconciler trusted the - // (stale) status field "A:0" as prior instead, current - // "A:0,C:0" would be a strict superset of "A:0" — no cascade. - // With the shadow as authoritative prior, prior="A:0,B:0" and - // the missing B IS detected as a replacement → cascade fires. - if err := f.r.Delete(context.Background(), added); err != nil { - t.Fatalf("delete added pod (OOM-kill): %v", err) - } - replacement := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-replacement-2", - Namespace: f.cacheNS, - UID: "cache-pod-uid-replacement-2", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), replacement); err != nil { - t.Fatalf("create replacement: %v", err) - } - if err := f.r.Status().Update(context.Background(), replacement); err != nil { - t.Fatalf("ready replacement: %v", err) - } - // Keep Deployment converged at replicas=2 throughout. - if err := setOwnedDeploymentConverged(f, 2); err != nil { - t.Fatalf("re-affirm converged at replicas=2: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-replacement wait = %v, want 0", wait) - } - f.reloadEngineDep(t) - wantFinal := serverInstanceID(f.serverPod) + "," + serverInstanceID(replacement) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != wantFinal { - t.Fatalf("cascade annotation = %q, want %q (shadow must override stale status as prior — without the override the missing-B transition would look like a strict superset and miss the cascade)", got, wantFinal) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter = %v, want exactly 1", got) - } -} - -// TestReconcileServerInstance_StaleDeploymentStatusDoesNotPersistMidpoint -// drives the race where the Deployment.Status counters lie about -// convergence: spec.replicas=1, status.readyReplicas=1, -// status.updatedReplicas=1, observedGeneration current — but the -// LIVE pod list has 2 Ready pods (a maxSurge mid-rollout where the -// apps/v1 Deployment controller has not yet observed the new pod). -// If the convergence check trusted only the Status counters, the -// strict-superset midpoint would be persisted as a "scale-up" -// baseline; a subsequent rollback dropping the new pod would then -// look like a real replacement and false-cascade the engine fleet. -// The cross-check against len(live Ready pods) closes that race. -func TestReconcileServerInstance_StaleDeploymentStatusDoesNotPersistMidpoint(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline: 1 Ready pod, Deployment converged at replicas=1. - if err := setOwnedDeploymentConverged(f, 1); err != nil { - t.Fatalf("set baseline Deployment converged: %v", err) - } - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Simulate a rolling-update midpoint where the apps controller's - // Status counters are STALE: they still claim readyReplicas=1 - // (matching spec.replicas=1) while the live pod list has 2 Ready - // pods. This is what stale-status convergence looks like to our - // reconciler. Leave Deployment.Status unchanged — it already - // reports {ReadyReplicas: 1, UpdatedReplicas: 1, replicas=1} - // from setOwnedDeploymentConverged(f, 1) above. - tru := true - newer := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-zzz", - Namespace: f.cacheNS, - UID: "cache-pod-uid-newer-stale-status", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), newer); err != nil { - t.Fatalf("create newer pod (stale-status midpoint): %v", err) - } - if err := f.r.Status().Update(context.Background(), newer); err != nil { - t.Fatalf("ready newer pod: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("midpoint wait = %v, want 0", wait) - } - f.reload(t) - // CRITICAL ASSERTION: the latch must NOT advance. Without the - // len(ready)==wantReplicas clause in the convergence check, the - // stale-status counters (which look converged at replicas=1) - // would convince the reconciler that the {old,new} pair is a - // steady-state scale-up rather than a transient midpoint, and - // the widened latch would get persisted. - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("midpoint ObservedServerInstance = %q, want %q (stale Deployment.Status counters reported convergence, but the live pod count (%d) > spec.replicas (1) is a midpoint, not a scale-up — the latch must NOT advance or a rollback would false-cascade)", got, baseline, 2) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("stale-status midpoint triggered a cascade; only converged transitions or real replacements should cascade") - } - - // Now simulate the rollback: the new pod is killed. The latch - // is still on the baseline, so prior == currentID == baseline, - // and the rollback is a true no-op. - if err := f.r.Delete(context.Background(), newer); err != nil { - t.Fatalf("delete rolled-back new pod: %v", err) - } - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-rollback wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("post-rollback ObservedServerInstance = %q, want %q (rolled-back rolling update must be a no-op; the original pod's process never changed)", got, baseline) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 0 { - t.Fatalf("cascade counter = %v, want 0 (no real cache-server replacement happened)", got) - } -} - -// setOwnedDeploymentConverged mutates the fixture's serverDep so its -// Status reflects a converged Deployment at the given replica count. -// Returns the apply error if any. Used by ConvergedScaleUp test. -func setOwnedDeploymentConverged(f *cascadeRestartFixture, replicas int32) error { - // Fetch a live copy (the fake client tracks resource versions). - live := &appsv1.Deployment{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverDep.Name, Namespace: f.serverDep.Namespace}, live); err != nil { - return fmt.Errorf("get live serverDep: %w", err) - } - r := replicas - live.Spec.Replicas = &r - if err := f.r.Update(context.Background(), live); err != nil { - return fmt.Errorf("update serverDep.Spec.Replicas: %w", err) - } - // Status subresource requires a separate update. - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverDep.Name, Namespace: f.serverDep.Namespace}, live); err != nil { - return fmt.Errorf("reload live serverDep: %w", err) - } - live.Status.ReadyReplicas = replicas - live.Status.UpdatedReplicas = replicas - live.Status.Replicas = replicas - live.Status.ObservedGeneration = live.Generation - if err := f.r.Status().Update(context.Background(), live); err != nil { - return fmt.Errorf("update serverDep.Status: %w", err) - } - return nil -} - -// TestReconcileServerInstance_InPlaceContainerRestartCascades asserts -// that an in-place container restart inside the cache-server pod -// (kubelet respawning a crashed container, e.g. on OOM with -// restartPolicy=Always — pod.UID stays the same) still advances -// observedServerInstance and triggers the cascade. The per-pod -// identifier sums containerStatuses[].restartCount, so a bump in any -// container's restart count changes the identifier without needing -// the pod to be replaced. -func TestReconcileServerInstance_InPlaceContainerRestartCascades(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline observation pins the identifier. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - if baseline != serverInstanceID(f.serverPod) { - t.Fatalf("baseline ObservedServerInstance = %q, want %q", baseline, serverInstanceID(f.serverPod)) - } - - // Simulate the kubelet bumping the lmcache-server container's - // restart count from 0 to 1 (e.g. OOM-killed container respawned - // in-place; same pod, same pod.UID, fresh LMCache process). - live := &corev1.Pod{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverPod.Name, Namespace: f.cacheNS}, live); err != nil { - t.Fatalf("get serverPod: %v", err) - } - live.Status.ContainerStatuses = []corev1.ContainerStatus{{ - Name: "lmcache-server", - Ready: true, - RestartCount: 1, - }} - if err := f.r.Status().Update(context.Background(), live); err != nil { - t.Fatalf("bump container restart count: %v", err) - } - - // Wait past the rate-limit window (fixture sets 50ms). - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-restart wait = %v, want 0", wait) - } - f.reload(t) - want := fmt.Sprintf("%s:1", f.serverPod.UID) - if got := f.backend.Status.ObservedServerInstance; got != want { - t.Fatalf("ObservedServerInstance after container restart = %q, want %q (the restart-count bump must advance the identifier)", got, want) - } - f.reloadEngineDep(t) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { - t.Fatalf("cascade annotation after container restart = %q, want %q", got, want) - } -} - -// TestReconcileServerInstance_SidecarRestartIgnored asserts that -// containerRunSum is scoped to the cache-server's own containers (per -// the owned Deployment's pod template), so a restart of an externally- -// injected sidecar (service mesh, Datadog, etc. — present in the -// pod's containerStatuses but absent from the Deployment template) -// does NOT advance observedServerInstance and does NOT cascade. A -// cascade for every Istio sidecar crash-loop would be a serious -// operator-facing regression. -func TestReconcileServerInstance_SidecarRestartIgnored(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline observation. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - baseline := f.backend.Status.ObservedServerInstance - - // Inject a sidecar restart event into the pod's containerStatuses. - // The sidecar is NOT in the owned Deployment's template, so the - // reconciler must ignore its restart count. - live := &corev1.Pod{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverPod.Name, Namespace: f.cacheNS}, live); err != nil { - t.Fatalf("get serverPod: %v", err) - } - live.Status.ContainerStatuses = []corev1.ContainerStatus{ - {Name: "lmcache-server", Ready: true, RestartCount: 0}, - {Name: "istio-proxy", Ready: true, RestartCount: 7}, - } - if err := f.r.Status().Update(context.Background(), live); err != nil { - t.Fatalf("inject sidecar restarts: %v", err) - } - - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-sidecar-restart wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != baseline { - t.Fatalf("ObservedServerInstance changed despite only sidecar restart: %q → %q", baseline, got) - } - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("sidecar restart triggered a cascade; only the cache-server's own containers should advance the identifier") - } -} - -// TestReconcileServerInstance_ForeignDeploymentSameNameIgnored asserts -// that when the backend's CacheBackend.UID does not control the live -// Deployment named after it (a foreign Deployment recreated under the -// same name, or operator drift), the reconciler refuses to attribute -// its pods to the backend and observedServerInstance stays empty. -func TestReconcileServerInstance_ForeignDeploymentSameNameIgnored(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Rewrite the cache-server Deployment's controller-owner ref to - // point at some OTHER CacheBackend (a foreign UID). - dep := f.serverDep.DeepCopy() - dep.OwnerReferences[0].UID = "foreign-cb-uid" - if err := f.r.Update(context.Background(), dep); err != nil { - t.Fatalf("rewrite owner ref: %v", err) - } - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != "" { - t.Fatalf("ObservedServerInstance = %q, want empty (foreign Deployment must not be attributed to this backend)", got) - } -} - -func TestReconcileServerInstance_NonInjectedPodsDoNotCascade(t *testing.T) { - f := newCascadeRestartFixture(t) - // Drop the injected-by annotation on the engine pod (e.g. webhook was - // unreachable at admission time). The Deployment matches the selector - // but the pod is NOT actually wired to this backend, so no cascade. - enginePod := f.enginePod.DeepCopy() - delete(enginePod.Annotations, podwebhook.AnnotationInjectedBy) - delete(enginePod.Annotations, podwebhook.AnnotationInjectedByUID) - if err := f.r.Update(context.Background(), enginePod); err != nil { - t.Fatalf("strip injected-by: %v", err) - } - - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - - f.reloadEngineDep(t) - if _, ok := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("engine deployment cascaded despite no injected-by annotation") - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - // We still count the cascade-call as one occurrence (an - // operator could otherwise miss flapping-server symptoms when - // no engines happen to be injected). Zero touched Deployments - // is documented in the metric Help text as a valid cascade. - t.Fatalf("cascade counter = %v, want 1 (a transition with zero matched Deployments is still one cascade event)", got) - } -} - -// TestReconcileServerInstance_SelfTargetGuardSkipsOwnDeployment -// drives the self-induced-rollout-loop scenario: an over-broad -// spec.engineSelector overlaps the cache-server pod's labels AND -// the pod webhook stamps the cache-server pod with the backend's -// injected-by + injected-by-uid annotations. Without the -// self-target guard, the cascade would pull the cache-server's own -// Deployment into the target set; annotating it would roll the -// cache-server, the controller would observe the new pod's UID, -// fire another cascade, and loop forever. The guard recognizes the -// canonical name (backend.Name == owned-Deployment name) and skips -// it. The engine Deployment must still be annotated — the guard -// must be narrow. -func TestReconcileServerInstance_SelfTargetGuardSkipsOwnDeployment(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Establish baseline. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - - // Stamp the cache-server pod with this backend's injected-by - // annotations — simulating the misconfiguration the guard - // defends against (webhook decided to inject the cache-server - // pod because its labels matched spec.engineSelector). - live := &corev1.Pod{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.serverPod.Name, Namespace: f.cacheNS}, live); err != nil { - t.Fatalf("get cache-server pod: %v", err) - } - if live.Annotations == nil { - live.Annotations = map[string]string{} - } - live.Annotations[podwebhook.AnnotationInjectedBy] = f.cacheNS + "/" + f.cacheName - live.Annotations[podwebhook.AnnotationInjectedByUID] = string(f.backend.UID) - if err := f.r.Update(context.Background(), live); err != nil { - t.Fatalf("stamp cache-server pod with injected-by annotations: %v", err) - } - - // Replace the cache-server pod to trigger a cascade. Use a new - // UID so the cascade decision fires. - if err := f.r.Delete(context.Background(), f.serverPod); err != nil { - t.Fatalf("delete old cache-server pod: %v", err) - } - tru := true - replacement := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-replacement", - Namespace: f.cacheNS, - UID: "cache-pod-uid-replacement", - Labels: selectorLabels(f.cacheName), - Annotations: map[string]string{ - // Replacement also carries the misconfigured - // injected-by stamp. - podwebhook.AnnotationInjectedBy: f.cacheNS + "/" + f.cacheName, - podwebhook.AnnotationInjectedByUID: string(f.backend.UID), - }, - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), replacement); err != nil { - t.Fatalf("create replacement cache-server pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), replacement); err != nil { - t.Fatalf("ready replacement: %v", err) - } - - // Wait past the rate-limit window. - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-replacement wait = %v, want 0", wait) - } - - // Self-target guard: the cache-server Deployment must NOT have - // been annotated, even though its pod carried the matching - // injected-by stamp. - ownDep := &appsv1.Deployment{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.cacheName, Namespace: f.cacheNS}, ownDep); err != nil { - t.Fatalf("get cache-server Deployment: %v", err) - } - if _, ok := ownDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; ok { - t.Fatalf("cache-server's own Deployment was annotated for cascade — self-induced rollout loop would follow") - } - - // The engine Deployment SHOULD still have been annotated — the - // guard must be narrow (only skip the backend's own Deployment). - f.reloadEngineDep(t) - want := serverInstanceID(replacement) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { - t.Fatalf("engine Deployment cascade annotation = %q, want %q (guard must be narrow — engine cascades still fire)", got, want) - } -} - -// TestReconcileServerInstance_ShadowRecoversBaselineFromPatchFailure -// drives the patch-failure-window scenario: a transient -// status-subresource patch failure on the first observation leaves -// status.observedServerInstance empty. Without the in-process shadow, -// a subsequent real cache-server replacement during that window would -// read prior="" and misclassify the replacement as another first -// observation (empty→set, no cascade) — engines stuck on stale -// sockets. With the shadow, the reconciler recovers the intended -// baseline from in-memory state and the replacement cascades. -func TestReconcileServerInstance_ShadowRecoversBaselineFromPatchFailure(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Wrap the client so the FIRST status-subresource patch returns a - // synthetic error (simulating a conflict / transient apiserver - // hiccup), then becomes transparent on subsequent attempts. - failOnce := &statusPatchFailingClient{Client: f.r.Client, remaining: 1} - f.r.Client = failOnce - - // First observation: status patch fails, latch stays "", but the - // shadow records the attempt. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait <= 0 { - t.Fatalf("first-observation patch failure should return a requeue duration; got %v", wait) - } - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != "" { - t.Fatalf("status.observedServerInstance = %q after simulated patch failure; want empty", got) - } - - // Now replace the cache-server pod with a new one (different UID) - // — simulating a server restart in the patch-failure window. - if err := f.r.Delete(context.Background(), f.serverPod); err != nil { - t.Fatalf("delete old server pod: %v", err) - } - tru := true - replacement := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-replacement", - Namespace: f.cacheNS, - UID: "cache-pod-uid-replacement", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), replacement); err != nil { - t.Fatalf("create replacement pod: %v", err) - } - if err := f.r.Status().Update(context.Background(), replacement); err != nil { - t.Fatalf("ready replacement pod: %v", err) - } - - // Wait past the rate-limit (fixture is 50ms) so the cascade is - // eligible on the next reconcile. - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("post-replacement wait = %v, want 0", wait) - } - - // The cascade must have fired. Without the shadow this would be a - // false-empty→set first-observation case (engines stranded). - f.reloadEngineDep(t) - want := serverInstanceID(replacement) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { - t.Fatalf("cascade annotation = %q, want %q (shadow must recover the lost baseline so the replacement cascades)", got, want) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter = %v, want 1 (replacement after a swallowed baseline-patch must still count as one cascade event)", got) - } -} - -// TestReconcileServerInstance_CounterIncrementsExactlyOncePerEvent -// drives the "one increment per cascade EVENT" contract through a -// failed-persist retry cycle. The cascade fires (annotates the -// engine) on the first attempt, and the counter advances at that -// point — engines are already recovering, so the metric should -// reflect the recovery regardless of whether the latch persist -// succeeded yet. On the subsequent retry the persist finally -// succeeds via the shadow short-circuit; the counter must NOT -// double-count because the `counted` map already holds this -// (key, currentID). -func TestReconcileServerInstance_CounterIncrementsExactlyOncePerEvent(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Baseline observation persists normally. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("baseline wait = %v, want 0", wait) - } - f.reload(t) - if baseline := f.backend.Status.ObservedServerInstance; baseline == "" { - t.Fatalf("baseline observedServerInstance is empty; expected the first observation to persist") - } - - // Replace the server pod. The next reconcile should cascade. - // Inject a status-patch-failing wrapper so the FIRST cascade- - // follow-up patch fails; the cascade itself (annotate engines) - // runs successfully, the counter advances, but the latch persist - // returns an error. - if err := f.r.Delete(context.Background(), f.serverPod); err != nil { - t.Fatalf("delete server pod: %v", err) - } - tru := true - replacement := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cache-pod-rep2", - Namespace: f.cacheNS, - UID: "cache-pod-uid-rep2", - Labels: selectorLabels(f.cacheName), - OwnerReferences: []metav1.OwnerReference{{ - APIVersion: "apps/v1", - Kind: "ReplicaSet", - Name: f.serverRS.Name, - UID: f.serverRS.UID, - Controller: &tru, - }}, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, - }, - } - if err := f.r.Create(context.Background(), replacement); err != nil { - t.Fatalf("create replacement: %v", err) - } - if err := f.r.Status().Update(context.Background(), replacement); err != nil { - t.Fatalf("ready replacement: %v", err) - } - failOnce := &statusPatchFailingClient{Client: f.r.Client, remaining: 1} - f.r.Client = failOnce - - time.Sleep(60 * time.Millisecond) - // First cascade attempt: annotates the engine + advances the - // counter, then the status patch fails. The counter MUST be at - // 1 by the end of this reconcile — engines are recovering, so - // the operator-visible metric should reflect the event even if - // the on-cluster latch could not be advanced yet. - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait <= 0 { - t.Fatalf("post-replacement first-attempt wait = %v, want positive (persist failed → request retry)", wait) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter after first attempt = %v, want 1 (engines were annotated; the metric must reflect the cascade event regardless of persist success)", got) - } - f.reloadEngineDep(t) - want := serverInstanceID(replacement) - if got := f.engineDep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != want { - t.Fatalf("engine annotation = %q, want %q (annotate must precede persist; engines should be recovering)", got, want) - } - - // Wait past the rate-limit window so canCascade lets the retry - // through. The status field still holds the PRE-cascade baseline - // (the first persist failed), so prior != currentID and the - // reconcile re-enters the cascade branch (not the shadow short- - // circuit). The cascade then runs idempotently: annotates are - // no-ops (the trigger already matches currentID), the counter - // does NOT advance (shouldIncrementCascade returns false because - // `counted` already holds (key, currentID)), and the persist - // finally succeeds. - time.Sleep(60 * time.Millisecond) - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("retry-reconcile wait = %v, want 0", wait) - } - if got := cascadeRestartsCount(t, f.cacheNS, f.cacheName, cascadeRestartReasonServerInstanceChanged); got != 1 { - t.Fatalf("cascade counter after persist retry = %v, want exactly 1 (one cascade event = one increment, regardless of how many persist retries were required)", got) - } - // And the status field is now in sync with the in-process baseline. - f.reload(t) - if got := f.backend.Status.ObservedServerInstance; got != want { - t.Fatalf("status.observedServerInstance after retry = %q, want %q (retry must reconcile the field)", got, want) - } -} - -// statusPatchFailingClient wraps a client.Client and returns a -// synthetic error from the FIRST `remaining` calls to Status().Patch -// on a CacheBackend, then passes calls through. Used to simulate a -// transient status-subresource patch failure window. -type statusPatchFailingClient struct { - client.Client - remaining int -} - -func (c *statusPatchFailingClient) Status() client.SubResourceWriter { - return &statusPatchFailingSubResource{ - SubResourceWriter: c.Client.Status(), - owner: c, - } -} - -type statusPatchFailingSubResource struct { - client.SubResourceWriter - owner *statusPatchFailingClient -} - -func (s *statusPatchFailingSubResource) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.SubResourcePatchOption) error { - if _, ok := obj.(*cachev1alpha1.CacheBackend); ok && s.owner.remaining > 0 { - s.owner.remaining-- - return fmt.Errorf("synthetic status-subresource patch failure for test") - } - return s.SubResourceWriter.Patch(ctx, obj, p, opts...) -} - -func TestReconcileServerInstance_AnnotateIdempotent(t *testing.T) { - f := newCascadeRestartFixture(t) - // Pre-seed the engine Deployment's pod template with the trigger - // annotation set to the CURRENT cache-server UID. The cascade should - // detect that and skip a no-op Patch (which would otherwise bump the - // rollout revision and pointlessly recycle engine pods). - dep := f.engineDep.DeepCopy() - if dep.Spec.Template.Annotations == nil { - dep.Spec.Template.Annotations = map[string]string{} - } - dep.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] = serverInstanceID(f.serverPod) - if err := f.r.Update(context.Background(), dep); err != nil { - t.Fatalf("preseed annotation: %v", err) - } - - f.backend.Status.ObservedServerInstance = "previous-server-uid" - if err := f.r.Status().Update(context.Background(), f.backend); err != nil { - t.Fatalf("seed status: %v", err) - } - f.reload(t) - - // Use a tracking client to confirm no second Deployment write occurs. - patches := 0 - tracked := &countingClient{Client: f.r.Client, patchCount: &patches} - f.r.Client = tracked - defer func() { f.r.Client = tracked.Client }() - - if wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend); wait != 0 { - t.Fatalf("wait = %v, want 0", wait) - } - if patches != 0 { - t.Fatalf("Deployment patch count = %d, want 0 (already up to date)", patches) - } -} - -func TestPodOwningDeployment_ResolvesViaReplicaSet(t *testing.T) { - f := newCascadeRestartFixture(t) - name, uid, ok, err := f.r.podOwningDeployment(context.Background(), f.r.Client, f.enginePod) - if err != nil { - t.Fatalf("podOwningDeployment err = %v, want nil", err) - } - if !ok { - t.Fatalf("podOwningDeployment ok = false, want true") - } - if name != f.engineDepN { - t.Fatalf("Deployment name = %q, want %q", name, f.engineDepN) - } - if uid == "" { - t.Fatalf("Deployment UID is empty; the owner-chain walk must return a non-empty UID so the cascade patch can re-verify identity") - } -} - -func TestPodOwningDeployment_NoOwnerReturnsFalse(t *testing.T) { - f := newCascadeRestartFixture(t) - bare := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "bare-pod", - Namespace: f.engineNS, - }, - } - if _, _, ok, err := f.r.podOwningDeployment(context.Background(), f.r.Client, bare); err != nil || ok { - t.Fatalf("podOwningDeployment for an unowned pod returned (ok=%v, err=%v); want (false, nil)", ok, err) - } -} - -// TestAnnotateDeploymentForCascade_TOCTOUDifferentUIDSkipsPatch locks -// the TOCTOU contract: if the live Deployment's UID does not match -// the UID observed during owner-chain resolution, the patch is -// skipped (the resolved target was deleted and re-created under the -// same name between resolution and annotate). A name-only patch in -// that window would roll an unrelated workload. -func TestAnnotateDeploymentForCascade_TOCTOUDifferentUIDSkipsPatch(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Look up the live Deployment UID so we can pass a *different* one. - live := &appsv1.Deployment{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.engineDepN, Namespace: f.engineNS}, live); err != nil { - t.Fatalf("get live engine Deployment: %v", err) - } - bogus := string(live.UID) + "-stale" - - beforeTrigger := live.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger] - - patched, err := f.r.annotateDeploymentForCascade( - context.Background(), - f.engineNS, - f.engineDepN, - bogus, // expectedUID — deliberately stale - "new-instance-id-xyz", // serverInstanceID - ) - if err != nil { - t.Fatalf("annotateDeploymentForCascade returned err = %v; want nil (TOCTOU skip is not an error)", err) - } - if patched { - t.Fatalf("annotateDeploymentForCascade patched = true; want false (UID mismatch should refuse the patch)") - } - - // Confirm the Deployment was NOT modified. - after := &appsv1.Deployment{} - if err := f.r.Get(context.Background(), types.NamespacedName{Name: f.engineDepN, Namespace: f.engineNS}, after); err != nil { - t.Fatalf("get post-call engine Deployment: %v", err) - } - if got := after.Spec.Template.Annotations[AnnotationCacheServerRestartTrigger]; got != beforeTrigger { - t.Fatalf("Deployment annotation was modified despite UID mismatch: before=%q after=%q", beforeTrigger, got) - } -} - -// TestReconcileServerInstance_ObservationErrorRequeues asserts that when -// currentServerInstanceID fails (transient apiserver/RBAC hiccup), -// reconcileServerInstance returns a positive requeue duration so the -// reconcile retries within the rate-limit window — without this, an -// observation failure would silently skip the cascade and the only -// path back is unrelated watch events. -func TestReconcileServerInstance_ObservationErrorRequeues(t *testing.T) { - f := newCascadeRestartFixture(t) - - // Inject a Client whose Pod List errors. The pod List in - // currentServerInstanceID is the first apiserver call on the - // reconciler's hot path, so any error here exercises the - // "observation failed" branch. - f.r.Client = &erroringPodListClient{Client: f.r.Client} - - wait := f.r.reconcileServerInstance(context.Background(), logr.Discard(), f.backend) - if wait <= 0 { - t.Fatalf("reconcileServerInstance wait = %v on observation failure; want a positive requeue (rate-limit interval)", wait) - } - if want := f.r.minServerRestartCascadeInterval(); wait != want { - t.Fatalf("reconcileServerInstance wait = %v on observation failure; want %v (the rate-limit interval)", wait, want) - } -} - -// erroringPodListClient wraps a client.Client and returns a synthetic -// error from List() when the target is a PodList. Used to exercise the -// reconcileServerInstance observation-failure path. -type erroringPodListClient struct { - client.Client -} - -func (c *erroringPodListClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - if _, ok := list.(*corev1.PodList); ok { - return fmt.Errorf("synthetic pod-list failure for test") - } - return c.Client.List(ctx, list, opts...) -} - -// countingClient counts Patch calls on Deployments. Used by -// TestReconcileServerInstance_AnnotateIdempotent to confirm a no-op -// cascade does not bump the rollout revision. -type countingClient struct { - client.Client - patchCount *int -} - -func (c *countingClient) Patch(ctx context.Context, obj client.Object, p client.Patch, opts ...client.PatchOption) error { - if _, ok := obj.(*appsv1.Deployment); ok { - *c.patchCount++ - } - return c.Client.Patch(ctx, obj, p, opts...) -} diff --git a/internal/controller/cachebackend_serverless.go b/internal/controller/cachebackend_serverless.go index 0af5b8c3..18bd18d9 100644 --- a/internal/controller/cachebackend_serverless.go +++ b/internal/controller/cachebackend_serverless.go @@ -17,12 +17,11 @@ import ( "time" ) -// reconcileExternal mirrors an externally owned backend's configured endpoint -// to status. For legacy backends, admission acceptance of the endpoint remains -// the only readiness signal because there is no Service to wait on. Typed -// PodLocal MP additionally aggregates the independently observed connector; -// endpoint acceptance alone must not report a missing/unhealthy native sidecar -// as Ready. +// reconcileExternal mirrors an externally owned provider's configured endpoint +// to status. Endpoint acceptance is the provider readiness signal because there +// is no Service to wait on. Typed PodLocal MP also aggregates the independently +// observed connector, so endpoint acceptance alone cannot report a missing or +// unhealthy native sidecar as Ready. // // Three terminal states, each driven by the SAME shape rule the // validating webhook applies on CREATE/UPDATE — so the reconciler is @@ -52,11 +51,6 @@ import ( // it on its own; an External backend whose engine pods still report KV events // legitimately keeps it). func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { - // Wipe the in-memory cascade shadow + rate-limit timestamp - // alongside the on-cluster status clearing below — see - // clearServerInstanceLatchShadow for why a lingering shadow - // across managed→External→managed would false-cascade. - r.clearServerInstanceLatchShadow(backend) // Wipe the functional-probe rate-limit entry alongside removing // the FunctionalProbeOK condition below. Without this, a CR that // flips managed → External → managed within the 30s rate-limit @@ -70,7 +64,7 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend // TrimSpace before every decision. Admission rejects a // whitespace-only endpoint at write time, but a pre-existing // CR in etcd from before admission was installed can still - // carry one. Publishing the trimmed value as status.endpoint + // carry one. Publishing the trimmed value in remote-storage status // means the pod webhook's `endpoint == ""` short-circuit // naturally catches whitespace too without a second TrimSpace // at the consumer. @@ -79,13 +73,6 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend if storage != nil { endpoint = strings.TrimSpace(storage.Endpoint) } - backend.Status.Endpoint = endpoint - // Clear the cache-server-instance latch — External backends - // have no controller-managed cache-server pods, and - // cleanupOwnedWorkload above has just deleted any prior - // managed Deployment. Leaving the latch set would expose a - // stale UID to operators. - backend.Status.ObservedServerInstance = "" backend.Status.ObservedGeneration = backend.Generation // Decide the Ready reason + message in one place so the @@ -171,7 +158,7 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend } // reconcileEventsOnly drives an events-only (tier-1 routing) backend: it -// provisions NO cache-server workload and publishes no endpoint, but still runs +// provisions no remote-provider workload and publishes no endpoint, but still runs // the KV-event readiness gate so Ready reflects "the engine is reporting state" // exactly as a managed backend does. The kvevent-subscriber sidecar is injected // engine-side by the (mode-aware) pod webhook, and status.indexParticipation is @@ -179,9 +166,9 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend // per-backend index slice behave identically to a managed backend; only the // offload tier (a server + KV connector) is absent. // -// Like reconcileExternal it clears the in-memory cascade shadow and the -// functional-probe rate-limit entry (there is no server to cascade-restart or -// probe) and clears the managed-only FunctionalProbeOK / T2Degraded conditions. +// Like reconcileExternal it clears the functional-probe rate-limit entry (there +// is no managed provider to probe) and clears the managed-only FunctionalProbeOK +// and T2Degraded conditions. // Events-only also clears EngineKernelsHealthy / EngineCompatibility because it // loads no connector; host-only evaluates both engine-side diagnostics normally. // The firstEventTimeout window is anchored on status.firstAvailableAt, latched @@ -203,9 +190,9 @@ func (r *CacheBackendReconciler) reconcileHostOnly(ctx context.Context, backend func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backend *cachev1alpha1.CacheBackend, activeReason, activeMessage string) (ctrl.Result, error) { now := time.Now() // A backend flipping INTO a serverless mode (events-only or host-only) from a - // server-bearing mode still carries that mode's status.endpoint / - // observedServerInstance at the top of this reconcile. Serverless modes clear - // both below, so a non-empty value here uniquely marks the first reconcile + // remote-storage mode may still carry that mode's status at the top of this + // reconcile. Serverless modes clear it below, so a non-empty value here + // uniquely marks the first reconcile // after the flip. // On that transition any latched firstAvailableAt reflects the OLD mode's // availability (e.g. an Offload workload that went Available long ago), not the @@ -213,7 +200,7 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen // anchor could breach the window the instant we flip and strand the backend at // NoKVEventsObserved/Degraded. Re-anchor to now so the new mode gets a fresh // first-event window from when it took effect. - transitionedFromServerMode := backend.Status.Endpoint != "" || backend.Status.ObservedServerInstance != "" + transitionedFromServerMode := backend.Status.RemoteStorage != nil // Base readiness is unconditionally True (no workload to gate on); the // KV-event gate layers AwaitingFirstKVEvent → KVEventsObserved / // NoKVEventsObserved on top, anchored on the firstAvailableAt latch. @@ -251,17 +238,11 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen } progressingStatus, progressingReason, progressingMessage := progressingFromReady(gate.readyStatus, gate.readyReason, gate.readyMessage) - // No server to cascade-restart or functionally probe — drop both in-memory - // trackers so a later Offload re-entry inside their windows starts clean - // (mirrors reconcileExternal). - r.clearServerInstanceLatchShadow(backend) + // No server to functionally probe; a later Offload re-entry starts clean. r.probeLimiter.forget(client.ObjectKeyFromObject(backend).String()) err := r.patchStatus(ctx, backend, func() { - // No provisioned server: no endpoint, no server-instance latch. - // status.indexParticipation stays poller-owned. - backend.Status.Endpoint = "" - backend.Status.ObservedServerInstance = "" + // No provisioned server. status.indexParticipation stays poller-owned. backend.Status.ObservedGeneration = backend.Generation if eventsOnly { backend.Status.RemoteStorage = nil @@ -363,9 +344,6 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { return err } - // Wipe the in-memory cascade shadow + rate-limit timestamp - // alongside the on-cluster status clearing below. - r.clearServerInstanceLatchShadow(backend) // Wipe the functional-probe rate-limit entry alongside removing // the FunctionalProbeOK condition below. Same reasoning as in // reconcileExternal — without this, a managed → Unmanaged → @@ -374,14 +352,8 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend // no fresh probe verdict. r.probeLimiter.forget(client.ObjectKeyFromObject(backend).String()) return r.patchStatus(ctx, backend, func() { - backend.Status.Endpoint = "" backend.Status.Connector = nil backend.Status.RemoteStorage = nil - // Clear the cache-server-instance latch — cleanupOwnedWorkload - // above has just deleted any prior managed Deployment and we - // no longer provision one, so a retained UID would advertise - // a stale identifier. - backend.Status.ObservedServerInstance = "" backend.Status.ObservedGeneration = backend.Generation meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeReady) meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeProgressing) @@ -403,7 +375,7 @@ func (r *CacheBackendReconciler) reconcileUnmanaged(ctx context.Context, backend // firstEventTimeout window. An unmanaged backend is not up in any sense, // so a stale anchor from a prior managed generation must not survive: // otherwise a later re-entry — in particular Offload→Unmanaged→EventsOnly, - // which clears endpoint/observedServerInstance so the events-only + // which clears remote-storage status so the events-only // re-anchor heuristic can't see the transition — would reuse a // long-past availability time and breach the window on the first // events-only reconcile. diff --git a/internal/controller/cachebackend_serverless_test.go b/internal/controller/cachebackend_serverless_test.go index f7d5261b..a0457e0c 100644 --- a/internal/controller/cachebackend_serverless_test.go +++ b/internal/controller/cachebackend_serverless_test.go @@ -6,30 +6,28 @@ package controller import ( "context" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" - "github.com/cachebox-project/inference-cache/internal/enginebinding" - podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" - adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" + "strings" + "testing" + appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "strings" - "testing" - "time" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" + "github.com/cachebox-project/inference-cache/internal/enginebinding" + podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" + adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" ) func TestReconcileCanonicalHostOnlyCacheCreatesNoProviderWorkload(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("host-only", "ns1") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.RemoteStorage = nil - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} r := newReconciler(scheme, cb) @@ -43,12 +41,12 @@ func TestReconcileCanonicalHostOnlyCacheCreatesNoProviderWorkload(t *testing.T) t.Fatalf("service lookup error = %v, want NotFound", err) } got := getBackend(t, r, cb.Name, cb.Namespace) - if got.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want empty for host-only hierarchy", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("status.remoteStorage = %+v, want nil for host-only hierarchy", got.Status.RemoteStorage) } ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != conditionReasonHostOnlyActive { - t.Fatalf("Ready = %+v, want True/%s", ready, conditionReasonHostOnlyActive) + if ready == nil || ready.Status != metav1.ConditionUnknown || ready.Reason != reasonConnectorUnverified { + t.Fatalf("Ready = %+v, want Unknown/%s until an engine Pod is observed", ready, reasonConnectorUnverified) } } @@ -61,22 +59,20 @@ func TestReconcileCanonicalSGLangHiCacheWithRemoteStorageIsUnmanaged(t *testing. Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, }, } r := newReconciler(scheme, cb) - reconcile(t, r, cb.Name, cb.Namespace) if _, err := getOptionalDeployment(t, r, cb.Name, cb.Namespace); !apierrors.IsNotFound(err) { t.Fatalf("deployment lookup error = %v, want NotFound", err) } got := getBackend(t, r, cb.Name, cb.Namespace) - if got.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want empty for unsupported binding", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("status.remoteStorage = %+v, want nil for unsupported binding", got.Status.RemoteStorage) } if ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady); ready != nil { t.Fatalf("unsupported binding published Ready condition: %+v", ready) @@ -87,29 +83,20 @@ func TestReconcileCanonicalHostOnlyCacheReportsEngineDiagnostics(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("host-only-kernel", "ns1") cb.UID = types.UID("host-only-kernel-uid") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "engine"}, - } + cb.Spec.RemoteStorage = nil + cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}} pod := strictPodWithKernelStatus(termed(1, enginebinding.KernelCheckMsgFailPrefix+" lmcache c_ops failed")) pod.ObjectMeta = metav1.ObjectMeta{ - Name: "engine", - Namespace: cb.Namespace, - Labels: map[string]string{"app": "engine"}, + Name: "engine", Namespace: cb.Namespace, Labels: map[string]string{"app": "engine"}, Annotations: map[string]string{ - podwebhook.AnnotationInjectedBy: cb.Namespace + "/" + cb.Name, - podwebhook.AnnotationInjectedByUID: string(cb.UID), + podwebhook.AnnotationInjectedBy: cb.Namespace + "/" + cb.Name, podwebhook.AnnotationInjectedByUID: string(cb.UID), }, } pod.Spec.Containers = []corev1.Container{{Name: "vllm"}} pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ - Name: "vllm", - State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ - Reason: crashLoopBackOffReason, - }}, + Name: "vllm", State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: crashLoopBackOffReason}}, }} r := newReconciler(scheme, cb, &pod) - reconcile(t, r, cb.Name, cb.Namespace) got := getBackend(t, r, cb.Name, cb.Namespace) @@ -117,33 +104,24 @@ func TestReconcileCanonicalHostOnlyCacheReportsEngineDiagnostics(t *testing.T) { if kernels == nil || kernels.Status != metav1.ConditionFalse || kernels.Reason != reasonKernelLoadFailed { t.Fatalf("EngineKernelsHealthy = %+v, want False/%s", kernels, reasonKernelLoadFailed) } - ready := meta.FindStatusCondition(got.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != reasonEngineKernelDegraded { - t.Fatalf("Ready = %+v, want False/%s", ready, reasonEngineKernelDegraded) - } compatibility := meta.FindStatusCondition(got.Status.Conditions, conditionTypeEngineCompatibility) - if compatibility == nil || compatibility.Status != metav1.ConditionFalse || - compatibility.Reason != reasonInjectedEngineCrashLooping { + if compatibility == nil || compatibility.Status != metav1.ConditionFalse || compatibility.Reason != reasonInjectedEngineCrashLooping { t.Fatalf("EngineCompatibility = %+v, want False/%s", compatibility, reasonInjectedEngineCrashLooping) } } -func TestReconcileTypeSwitchToExternalCleansUpChildren(t *testing.T) { +func TestReconcileManagedToExternalCleansUpProviderChildren(t *testing.T) { scheme := newScheme(t) r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - reconcile(t, r, "cache", "ns1") - // Child workload exists. if _, err := getOptionalDeployment(t, r, "cache", "ns1"); err != nil { - t.Fatalf("expected deployment after managed reconcile: %v", err) + t.Fatalf("expected managed Redis deployment: %v", err) } live := getBackend(t, r, "cache", "ns1") - live.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - live.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - live.Spec.RemoteStorage = externalLMCacheStorage("external.ns1.svc:8080") + live.Spec.RemoteStorage = externalRedisStorage("external.ns1.svc:6379") if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("switch to external: %v", err) + t.Fatalf("switch to external Redis: %v", err) } reconcile(t, r, "cache", "ns1") @@ -161,272 +139,22 @@ func TestReconcileTypeSwitchToExternalCleansUpChildren(t *testing.T) { if len(svcs.Items) != 0 { t.Fatalf("services = %d, want 0 after switch to External", len(svcs.Items)) } - if got := getBackend(t, r, "cache", "ns1").Status.Endpoint; got != "external.ns1.svc:8080" { - t.Fatalf("status.endpoint = %q, want mirrored external endpoint", got) - } -} - -// TestReconcileTypeSwitchToExternalClearsObservedServerInstance asserts -// that status.observedServerInstance is cleared when a managed -// CacheBackend transitions to External — leaving a stale latch on an -// External backend would surface a UID that no longer maps to any -// controller-managed pod, and a subsequent flip back to managed -// would inherit the stale baseline and either false-cascade -// immediately or false-pin a non-existent pod set. This is the -// lifecycle contract reconcileExternal encodes; a status-field flip -// is exactly the kind of seam tests must hold, alongside the -// preserved-fields contract (firstKVEventObservedAt must survive). -func TestReconcileTypeSwitchToExternalClearsObservedServerInstance(t *testing.T) { - scheme := newScheme(t) - r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - - reconcile(t, r, "cache", "ns1") - // Plant BOTH a baseline ObservedServerInstance AND an in-memory - // shadow value, simulating a managed period that had observed a - // Ready cache-server pod. The test then verifies that the - // External transition clears BOTH — without the planted shadow, - // the shadow assertion would vacuously pass on an empty map. - live := getBackend(t, r, "cache", "ns1") - live.Status.ObservedServerInstance = "cache-pod-uid:0" - if err := r.Status().Update(context.Background(), live); err != nil { - t.Fatalf("plant baseline observedServerInstance: %v", err) - } - plantedKey := cascadeKey{namespace: live.Namespace, name: live.Name, uid: string(live.UID)} - r.serverInstanceCascade.recordAttempt(plantedKey, "cache-pod-uid:0") - if got := r.serverInstanceCascade.lastAttempt(plantedKey); got != "cache-pod-uid:0" { - t.Fatalf("planted shadow precondition failed: lastAttempt = %q, want %q (test would be vacuous without a planted value)", got, "cache-pod-uid:0") - } - - // Confirm preserved fields we expect NOT to be clobbered alongside - // the latch (firstKVEventObservedAt + indexParticipation must - // survive the External transition per reconcileExternal's godoc). - preserved := getBackend(t, r, "cache", "ns1") - preserved.Status.FirstKVEventObservedAt = &metav1.Time{Time: time.Unix(1_000_000_000, 0).UTC()} - if err := r.Status().Update(context.Background(), preserved); err != nil { - t.Fatalf("plant firstKVEventObservedAt: %v", err) - } - - // Switch to External. - switching := getBackend(t, r, "cache", "ns1") - switching.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - switching.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - switching.Spec.RemoteStorage = externalLMCacheStorage("external.ns1.svc:8080") - if err := r.Update(context.Background(), switching); err != nil { - t.Fatalf("switch to external: %v", err) - } - reconcile(t, r, "cache", "ns1") - - got := getBackend(t, r, "cache", "ns1") - if got.Status.ObservedServerInstance != "" { - t.Fatalf("status.observedServerInstance = %q, want cleared on managed→External transition", got.Status.ObservedServerInstance) - } - if got.Status.FirstKVEventObservedAt == nil { - t.Fatalf("status.firstKVEventObservedAt was clobbered on External transition; it must survive as a monotonic latch") - } - // The in-memory cascade shadow MUST also be cleared. A retained - // shadow would let a later External→managed transition resolve - // effectivePrior to the prior-period currentID and false-cascade - // the engine fleet on the first new Ready pod. - if shadow := r.serverInstanceCascade.lastAttempt(plantedKey); shadow != "" { - t.Fatalf("cascade shadow = %q after managed→External transition; want cleared (a lingering shadow would false-cascade on the return path)", shadow) - } -} - -// TestReconcileSwitchToStatefulSetClearsObservedServerInstance asserts -// the same clearing for the managed→unsupported-runtime transition -// (reconcileUnmanaged path). The StatefulSet deployment-kind is -// currently the canonical unmanaged trigger. -func TestReconcileSwitchToStatefulSetClearsObservedServerInstance(t *testing.T) { - scheme := newScheme(t) - r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - - reconcile(t, r, "cache", "ns1") - live := getBackend(t, r, "cache", "ns1") - live.Status.ObservedServerInstance = "cache-pod-uid:0" - // Plant a stale KV-event-gate anchor too: the unmanaged transition must - // reset it so a later re-entry (managed or events-only) starts a fresh - // firstEventTimeout window rather than reusing this pre-unmanaged time. - staleAnchor := metav1.NewTime(time.Now().Add(-time.Hour)) - live.Status.FirstAvailableAt = &staleAnchor - if err := r.Status().Update(context.Background(), live); err != nil { - t.Fatalf("plant baseline observedServerInstance: %v", err) - } - plantedKey := cascadeKey{namespace: live.Namespace, name: live.Name, uid: string(live.UID)} - r.serverInstanceCascade.recordAttempt(plantedKey, "cache-pod-uid:0") - if got := r.serverInstanceCascade.lastAttempt(plantedKey); got != "cache-pod-uid:0" { - t.Fatalf("planted shadow precondition failed: lastAttempt = %q, want %q", got, "cache-pod-uid:0") - } - - switching := getBackend(t, r, "cache", "ns1") - switching.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet - if err := r.Update(context.Background(), switching); err != nil { - t.Fatalf("switch to StatefulSet: %v", err) - } - reconcile(t, r, "cache", "ns1") - - got := getBackend(t, r, "cache", "ns1") - if got.Status.ObservedServerInstance != "" { - t.Fatalf("status.observedServerInstance = %q, want cleared on managed→unmanaged transition", got.Status.ObservedServerInstance) - } - // In-memory shadow must also be cleared on the unmanaged path. - if shadow := r.serverInstanceCascade.lastAttempt(plantedKey); shadow != "" { - t.Fatalf("cascade shadow = %q after managed→unmanaged transition; want cleared", shadow) - } - // The stale KV-event-gate anchor must be reset — otherwise an Offload→ - // Unmanaged→EventsOnly path (which clears endpoint/observedServerInstance, - // so the events-only re-anchor heuristic can't detect the transition) would - // reuse this pre-unmanaged time and breach the first-event window instantly. - if got.Status.FirstAvailableAt != nil { - t.Fatalf("status.firstAvailableAt = %v, want cleared on managed→unmanaged transition", got.Status.FirstAvailableAt) - } -} - -func TestReconcileSwitchToSGLangHiCacheCleansManagedState(t *testing.T) { - scheme := newScheme(t) - managed := lmcacheBackend("cache", "ns1") - managed.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: ptrInt32(1), - MaxReplicas: 3, - } - r := newReconciler( - scheme, - managed, - &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-0", Namespace: "ns1", Labels: map[string]string{"app": "sglang"}}}, - &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "sglang-1", Namespace: "ns1", Labels: map[string]string{"app": "sglang"}}}, - ) - reconcile(t, r, "cache", "ns1") - var managedHPA autoscalingv2.HorizontalPodAutoscaler - if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, &managedHPA); err != nil { - t.Fatalf("get managed HPA before switch: %v", err) - } - - live := getBackend(t, r, "cache", "ns1") - matched := int32(2) - live.Status.Endpoint = "cache.ns1.svc:65432" - live.Status.ObservedServerInstance = "cache-pod-uid:0" - live.Status.MatchedEnginePods = &matched - live.Status.IndexParticipation = &cachev1alpha1.CacheBackendIndexParticipation{PrefixCount: 7} - meta.SetStatusCondition(&live.Status.Conditions, metav1.Condition{ - Type: conditionTypeReady, - Status: metav1.ConditionTrue, - Reason: "Available", - }) - if err := r.Status().Update(context.Background(), live); err != nil { - t.Fatalf("plant managed status: %v", err) - } - - switching := getBackend(t, r, "cache", "ns1") - switching.Generation = 2 - switching.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache - switching.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet - switching.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - } - switching.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "sglang"}, - } - switching.Spec.Autoscaling = nil - switching.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"} - if err := r.Update(context.Background(), switching); err != nil { - t.Fatalf("switch to SGLangHiCache: %v", err) - } - reconcile(t, r, "cache", "ns1") - - if _, err := getOptionalDeployment(t, r, "cache", "ns1"); !apierrors.IsNotFound(err) { - t.Fatalf("managed Deployment still exists after switch: %v", err) - } - var service corev1.Service - if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, &service); !apierrors.IsNotFound(err) { - t.Fatalf("managed Service still exists after switch: %v", err) - } - var hpa autoscalingv2.HorizontalPodAutoscaler - if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, &hpa); !apierrors.IsNotFound(err) { - t.Fatalf("managed HPA still exists after switch: %v", err) - } got := getBackend(t, r, "cache", "ns1") - if got.Status.Endpoint != "" || got.Status.ObservedServerInstance != "" { - t.Fatalf("stale managed endpoint/server instance survived: %+v", got.Status) - } - if len(got.Status.Conditions) != 0 { - t.Fatalf("SGLangHiCache first commit must publish no conditions, got %v", got.Status.Conditions) - } - if got.Status.ObservedGeneration != got.Generation { - t.Fatalf("observedGeneration = %d, want generation %d", got.Status.ObservedGeneration, got.Generation) - } - if got.Status.MatchedEnginePods == nil || *got.Status.MatchedEnginePods != 2 { - t.Fatalf("matchedEnginePods = %v, want preserved 2", got.Status.MatchedEnginePods) - } - if got.Status.IndexParticipation == nil || got.Status.IndexParticipation.PrefixCount != 7 { - t.Fatalf("indexParticipation = %+v, want preserved", got.Status.IndexParticipation) - } -} - -func TestReconcileStatefulSetKindDeferred(t *testing.T) { - scheme := newScheme(t) - cb := lmcacheBackend("cache", "ns1") - cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet - r := newReconciler(scheme, cb) - - reconcile(t, r, "cache", "ns1") - - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) - } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 (StatefulSet kind deferred — managed Deployments only for now)", len(deps.Items)) - } -} - -func TestReconcileSwitchToStatefulSetClearsStaleStatus(t *testing.T) { - scheme := newScheme(t) - r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - - reconcile(t, r, "cache", "ns1") - if ep := getBackend(t, r, "cache", "ns1").Status.Endpoint; ep == "" { - t.Fatalf("expected a published endpoint after managed reconcile") - } - - live := getBackend(t, r, "cache", "ns1") - live.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("switch to StatefulSet kind: %v", err) - } - reconcile(t, r, "cache", "ns1") - - updated := getBackend(t, r, "cache", "ns1") - if updated.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want cleared after no longer managed", updated.Status.Endpoint) - } - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond != nil { - t.Fatalf("Ready condition = %+v, want removed", cond) - } - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) - } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 after switch to StatefulSet kind", len(deps.Items)) + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "external.ns1.svc:6379" { + t.Fatalf("status.remoteStorage = %+v, want mirrored external endpoint", got.Status.RemoteStorage) } } func TestReconcileExternalAdvancesObservedGeneration(t *testing.T) { scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "default", Generation: 7}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("external.default.svc:8080"), - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: "external.default.svc:8080"}, + cb := lmcacheBackend("ext", "default") + cb.Generation = 7 + cb.Spec.RemoteStorage = externalRedisStorage("external.default.svc:6379") + cb.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "external.default.svc:6379", } r := newReconciler(scheme, cb) - reconcile(t, r, "ext", "default") - - // Endpoint is unchanged, but observedGeneration must still advance. if got := getBackend(t, r, "ext", "default").Status.ObservedGeneration; got != 7 { t.Fatalf("status.observedGeneration = %d, want 7", got) } @@ -434,16 +162,12 @@ func TestReconcileExternalAdvancesObservedGeneration(t *testing.T) { func TestReconcileUnmanagedTypeNoop(t *testing.T) { scheme := newScheme(t) - // An arbitrary unsupported value exercises the admission-bypassed - // "unsupported type → reconcileUnmanaged" path. cb := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1"}, Spec: cachev1alpha1.CacheBackendSpec{Type: cachev1alpha1.CacheBackendType("unsupported")}, } r := newReconciler(scheme, cb) - reconcile(t, r, "cache", "ns1") - var deps appsv1.DeploymentList if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { t.Fatalf("list deployments: %v", err) @@ -454,460 +178,120 @@ func TestReconcileUnmanagedTypeNoop(t *testing.T) { } func TestReconcileEventsOnlyUnsupportedPairIsUnmanaged(t *testing.T) { - // An EventsOnly backend whose (engine, type) pair has no registered - // adapter must reconcile as UNMANAGED, NOT as active events-only. - // Admission rejects an unsupported pair at write time, but a - // stored/admission-bypassed CR reaching the controller must not be - // advertised as a working routing tier: the pod webhook can't select an - // adapter for an unsupported pair, so it could never inject the - // kvevent-subscriber and no KV event would ever flow. dispatch confirms an - // adapter is selectable before routing to reconcileEventsOnly; on failure it - // falls to reconcileUnmanaged. The arbitrary value below is the unsupported - // fixture. scheme := newScheme(t) cb := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1", Generation: 1}, Spec: cachev1alpha1.CacheBackendSpec{ - Type: cachev1alpha1.CacheBackendType("unsupported"), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - }, + Type: cachev1alpha1.CacheBackendType("unsupported"), + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly}, }, } r := newReconciler(scheme, cb) - reconcile(t, r, "cache", "ns1") - got := getBackend(t, r, "cache", "ns1") - // reconcileUnmanaged removes the Ready / Progressing conditions; the - // events-only path (reconcileEventsOnly) would have PUBLISHED them. Their - // absence is the discriminator between "reconciled as unmanaged" and - // "reconciled as active events-only". if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("unsupported-pair events-only must NOT publish Ready (unmanaged path); got %+v", ready) - } - if prog := findCondition(got.Status.Conditions, conditionTypeProgressing); prog != nil { - t.Fatalf("unsupported-pair events-only must NOT publish Progressing (unmanaged path); got %+v", prog) - } - // And no workload is provisioned (unmanaged sheds everything). - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) - } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 for unmanaged events-only", len(deps.Items)) + t.Fatalf("unsupported-pair EventsOnly published Ready: %+v", ready) } } -func TestReconcileEventsOnlyAdapterRejectingHostOnlyBindingIsUnmanaged(t *testing.T) { +func TestReconcileEventsOnlyAdapterRejectingNilBindingIsUnmanaged(t *testing.T) { scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1", Generation: 1}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - }, - }, - } + cb := lmcacheBackend("cache", "ns1") + cb.Spec.RemoteStorage = nil + cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly r := newReconciler(scheme, cb) r.Registry = adapterruntime.NewRegistry() - r.Registry.Register(remoteOnlyRuntimeAdapter{KVCacheRuntimeAdapter: builtinruntime.NewVLLMLMCacheAdapter(builtinruntime.SubscriberConfig{})}) - + r.Registry.Register(remoteOnlyRuntimeAdapter{KVCacheRuntimeAdapter: builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{})}) reconcile(t, r, "cache", "ns1") - - got := getBackend(t, r, "cache", "ns1") - if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("events-only adapter rejecting nil binding must not publish Ready; got %+v", ready) - } - if prog := findCondition(got.Status.Conditions, conditionTypeProgressing); prog != nil { - t.Fatalf("events-only adapter rejecting nil binding must not publish Progressing; got %+v", prog) + if ready := findCondition(getBackend(t, r, "cache", "ns1").Status.Conditions, conditionTypeReady); ready != nil { + t.Fatalf("adapter rejecting nil binding published Ready: %+v", ready) } } func TestReconcileEventsOnlyTakesPrecedenceOverExternal(t *testing.T) { - // An admission-bypassed object that sets both externally owned remote storage - // and integration.mode=EventsOnly must reconcile via the events-only path. - // Admission rejects this pair, so this is defense-in-depth for stored CRs. If - // it reconciled as external storage it would publish an endpoint and allow KV - // connector injection, violating events-only's "no connector, no server" - // contract. The vLLM/LMCache pair has a registered adapter, so - // the events-only adapter-selectability check passes and the events-only - // reconcile runs. scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1", Generation: 1}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("external-cache.ns1.svc:8200"), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - }, - }, - } + cb := lmcacheBackend("cache", "ns1") + cb.Spec.RemoteStorage = externalRedisStorage("external-cache.ns1.svc:6379") + cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly r := newReconciler(scheme, cb) - r.Registry = adapterruntime.NewRegistry() - r.Registry.Register(builtinruntime.NewVLLMLMCacheAdapter(builtinruntime.SubscriberConfig{})) - reconcile(t, r, "cache", "ns1") got := getBackend(t, r, "cache", "ns1") - - // status.endpoint stays EMPTY — events-only publishes no endpoint. The - // external-ownership path would have mirrored - // spec.remoteStorage.endpoint here. - if got.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want empty (events-only wins over External; no endpoint mirrored)", got.Status.Endpoint) + if got.Status.RemoteStorage != nil { + t.Fatalf("status.remoteStorage = %+v, want nil because EventsOnly wins", got.Status.RemoteStorage) } - - // Ready is published by the events-only gate (AwaitingFirstKVEvent before any - // event), NOT by the external-ownership path - // (ExternalEndpointAccepted). The reason is - // the discriminator between the two reconcile paths. ready := findCondition(got.Status.Conditions, conditionTypeReady) - if ready == nil { - t.Fatalf("events-only must publish Ready; conditions = %v", got.Status.Conditions) - } - if ready.Reason == conditionReasonExternalEndpointAccepted { - t.Fatalf("Ready reason = %q — reconciled as External, but EventsOnly must take precedence", ready.Reason) - } - if ready.Status != metav1.ConditionFalse || ready.Reason != reasonAwaitingFirstKVEvent { - t.Fatalf("Ready = %+v, want False/AwaitingFirstKVEvent (events-only gate before any KV event)", ready) - } - - // No workload is provisioned (events-only is server-less). - var deps appsv1.DeploymentList - if err := r.List(context.Background(), &deps, client.InNamespace("ns1")); err != nil { - t.Fatalf("list deployments: %v", err) - } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 for events-only", len(deps.Items)) - } - var svcs corev1.ServiceList - if err := r.List(context.Background(), &svcs, client.InNamespace("ns1")); err != nil { - t.Fatalf("list services: %v", err) - } - if len(svcs.Items) != 0 { - t.Fatalf("services = %d, want 0 for events-only", len(svcs.Items)) + if ready == nil || ready.Reason == conditionReasonExternalEndpointAccepted { + t.Fatalf("EventsOnly did not take precedence; Ready = %+v", ready) } } -func TestReconcileExternalMirrorsEndpointToStatus(t *testing.T) { +func TestReconcileExternalMirrorsRedisEndpointAndSetsReady(t *testing.T) { scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("external-cache.default.svc:8080"), - }, - } + cb := lmcacheBackend("example", "default") + cb.Generation = 3 + cb.Spec.RemoteStorage = externalRedisStorage("external-cache.default.svc:6379") r := newReconciler(scheme, cb) - reconcile(t, r, "example", "default") - if got := getBackend(t, r, "example", "default").Status.Endpoint; got != "external-cache.default.svc:8080" { - t.Fatalf("status.endpoint = %q, want spec.remoteStorage.endpoint", got) - } -} - -func TestReconcileExternalSetsReadyTrue(t *testing.T) { - // Admission accepts spec.remoteStorage.endpoint for external ownership at - // write time, so the - // readiness signal is "operator says this endpoint exists and we - // accepted it" — there's no Service to wait on. Consumers (the - // future readiness gate, kubectl get cb, the indexParticipation - // poller for External) must see Ready=True. - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: "default", Generation: 3}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("ext.default.svc:8080"), - }, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, "ext", "default") - - got := getBackend(t, r, "ext", "default") - ready := findCondition(got.Status.Conditions, "Ready") - if ready == nil { - t.Fatalf("Ready condition missing; conditions = %v", got.Status.Conditions) - } - if ready.Status != metav1.ConditionTrue { - t.Fatalf("Ready status = %q, want %q", ready.Status, metav1.ConditionTrue) - } - if ready.Reason != "ExternalEndpointAccepted" { - t.Fatalf("Ready reason = %q, want ExternalEndpointAccepted", ready.Reason) - } - if ready.ObservedGeneration != 3 { - t.Fatalf("Ready.observedGeneration = %d, want 3", ready.ObservedGeneration) + got := getBackend(t, r, "example", "default") + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "external-cache.default.svc:6379" || got.Status.RemoteStorage.Ready != metav1.ConditionTrue { + t.Fatalf("status.remoteStorage = %+v, want ready mirrored Redis endpoint", got.Status.RemoteStorage) } - progressing := findCondition(got.Status.Conditions, "Progressing") - if progressing == nil { - t.Fatalf("Progressing condition missing; conditions = %v", got.Status.Conditions) + remoteReady := findCondition(got.Status.Conditions, conditionTypeRemoteStorageReady) + if remoteReady == nil || remoteReady.Status != metav1.ConditionTrue || remoteReady.Reason != reasonRemoteStorageReady { + t.Fatalf("RemoteStorageReady = %+v, want True/%s", remoteReady, reasonRemoteStorageReady) } - if progressing.Status != metav1.ConditionFalse { - t.Fatalf("Progressing status = %q, want %q", progressing.Status, metav1.ConditionFalse) + ready := findCondition(got.Status.Conditions, conditionTypeReady) + if ready == nil || ready.Status != metav1.ConditionUnknown || ready.Reason != reasonConnectorUnverified || ready.ObservedGeneration != 3 { + t.Fatalf("Ready = %+v, want Unknown/%s generation 3", ready, reasonConnectorUnverified) } } -func TestReconcileExternalInvalidEndpointSetsReadyFalse(t *testing.T) { - // An externally owned CR with a non-empty but malformed - // spec.remoteStorage.endpoint must - // be marked Ready=False/ExternalEndpointInvalid — current admission - // rejects these at write time, but a CR stored before the shape - // rule shipped can still carry e.g. `https://...`. Without this, - // the controller would advertise the broken value as Ready=True - // and the pod webhook would inject a URL the engine can't parse. - scheme := newScheme(t) - for _, tc := range []struct { - name, endpoint string - }{ - {"bad-scheme", "https://cache.example.com:443/api"}, - {"portless-host", "cache.example.com"}, - {"non-numeric-port", "cache.example.com:not-a-port"}, - {"zero-port", "cache.example.com:0"}, - {"out-of-range-port", "cache.example.com:70000"}, - {"unbracketed-ipv6", "2001:db8::1"}, - {"embedded-whitespace", "cache example:8200"}, +func TestReconcileExternalInvalidRedisEndpointSetsReadyFalse(t *testing.T) { + for _, endpoint := range []string{ + "https://cache.example.com:443/api", "cache.example.com", "cache.example.com:not-a-port", + "cache.example.com:0", "cache.example.com:70000", "2001:db8::1", "cache example:6379", } { - t.Run(tc.name, func(t *testing.T) { - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-bad", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(tc.endpoint), - }, - } + t.Run(endpoint, func(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("ext-bad", "default") + cb.Spec.RemoteStorage = externalRedisStorage(endpoint) r := newReconciler(scheme, cb) - reconcile(t, r, "ext-bad", "default") + reconcile(t, r, cb.Name, cb.Namespace) - got := getBackend(t, r, "ext-bad", "default") - ready := findCondition(got.Status.Conditions, "Ready") - if ready == nil || ready.Status != metav1.ConditionFalse { - t.Fatalf("Ready condition = %+v, want Status=False", ready) - } - if ready.Reason != "ExternalEndpointInvalid" { - t.Fatalf("Ready reason = %q, want ExternalEndpointInvalid", ready.Reason) + got := getBackend(t, r, cb.Name, cb.Namespace) + remoteReady := findCondition(got.Status.Conditions, conditionTypeRemoteStorageReady) + if remoteReady == nil || remoteReady.Status != metav1.ConditionFalse || remoteReady.Reason != conditionReasonExternalEndpointInvalid { + t.Fatalf("RemoteStorageReady = %+v, want False/%s", remoteReady, conditionReasonExternalEndpointInvalid) } - if !strings.Contains(ready.Message, "spec.remoteStorage.endpoint") { - t.Fatalf("Ready message = %q, want canonical field spec.remoteStorage.endpoint", ready.Message) + if !strings.Contains(remoteReady.Message, "spec.remoteStorage.endpoint") { + t.Fatalf("RemoteStorageReady message = %q, want canonical field", remoteReady.Message) } }) } } -func TestReconcileCanonicalExternalInvalidEndpointNamesCanonicalField(t *testing.T) { - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "canonical-ext-bad", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "https://cache.example.com:443/api", - }, - }, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, cb.Name, cb.Namespace) - - got := getBackend(t, r, cb.Name, cb.Namespace) - ready := findCondition(got.Status.Conditions, "Ready") - if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "ExternalEndpointInvalid" { - t.Fatalf("Ready condition = %+v, want False/ExternalEndpointInvalid", ready) - } - if !strings.Contains(ready.Message, "spec.remoteStorage.endpoint") { - t.Fatalf("Ready message = %q, want canonical field spec.remoteStorage.endpoint", ready.Message) - } - if strings.Contains(ready.Message, "spec.endpoint") { - t.Fatalf("Ready message = %q, must not name deprecated spec.endpoint", ready.Message) - } -} - -func TestReconcileCanonicalExternalEndpointUsesProviderProtocol(t *testing.T) { - scheme := newScheme(t) - tests := []struct { - name string - runtime cachev1alpha1.CacheBackendRuntime - provider cachev1alpha1.CacheBackendRemoteStorageProvider - endpoint string - wantStatus metav1.ConditionStatus - wantReason string - }{ - { - name: "redis rejects lm scheme", - runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - endpoint: "lm://redis.example:6379", - wantStatus: metav1.ConditionFalse, - wantReason: conditionReasonExternalEndpointInvalid, - }, - { - name: "mooncake accepts explicit scheme", - runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - endpoint: "mooncakestore://cache.example:50051", - wantStatus: metav1.ConditionTrue, - wantReason: conditionReasonExternalEndpointAccepted, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "external", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: tt.runtime, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: tt.provider, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: tt.endpoint, - }, - }, +func TestReconcileExternalMissingRedisEndpointClearsStatus(t *testing.T) { + for _, endpoint := range []string{"", " \t "} { + t.Run(endpoint, func(t *testing.T) { + scheme := newScheme(t) + cb := lmcacheBackend("ext-missing", "default") + cb.Spec.RemoteStorage = externalRedisStorage(endpoint) + cb.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Endpoint: "stale.default.svc:6379", Ready: metav1.ConditionTrue, } r := newReconciler(scheme, cb) reconcile(t, r, cb.Name, cb.Namespace) - ready := findCondition(getBackend(t, r, cb.Name, cb.Namespace).Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != tt.wantStatus || ready.Reason != tt.wantReason { - t.Fatalf("Ready = %+v, want %s/%s", ready, tt.wantStatus, tt.wantReason) + got := getBackend(t, r, cb.Name, cb.Namespace) + if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "" || got.Status.RemoteStorage.Ready != metav1.ConditionFalse { + t.Fatalf("status.remoteStorage = %+v, want present but not ready and empty endpoint", got.Status.RemoteStorage) + } + remoteReady := findCondition(got.Status.Conditions, conditionTypeRemoteStorageReady) + if remoteReady == nil || remoteReady.Status != metav1.ConditionFalse || remoteReady.Reason != conditionReasonExternalEndpointMissing { + t.Fatalf("RemoteStorageReady = %+v, want False/%s", remoteReady, conditionReasonExternalEndpointMissing) } }) } } - -func TestReconcileCanonicalExternalUnsupportedBindingStaysUnmanaged(t *testing.T) { - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "external-redis", Namespace: "default", Generation: 2}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "redis.example:6379", - }, - }, - Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "stale.example:6379", - Conditions: []metav1.Condition{{ - Type: conditionTypeReady, - Status: metav1.ConditionTrue, - Reason: conditionReasonExternalEndpointAccepted, - }}, - }, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, cb.Name, cb.Namespace) - - got := getBackend(t, r, cb.Name, cb.Namespace) - if got.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want cleared for unsupported external binding", got.Status.Endpoint) - } - if ready := findCondition(got.Status.Conditions, conditionTypeReady); ready != nil { - t.Fatalf("Ready = %+v, want absent for unmanaged unsupported external binding", ready) - } - if got.Status.ObservedGeneration != cb.Generation { - t.Fatalf("status.observedGeneration = %d, want %d", got.Status.ObservedGeneration, cb.Generation) - } -} - -func TestReconcileExternalEmptyEndpointSetsReadyFalse(t *testing.T) { - // Admission rejects this case at the webhook, but a CR already in etcd - // from before the webhook was installed must still publish a visible - // Ready=False so operators can see why the CR isn't usable instead of - // finding the condition simply absent. - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-no-ep", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(""), - }, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, "ext-no-ep", "default") - - got := getBackend(t, r, "ext-no-ep", "default") - ready := findCondition(got.Status.Conditions, "Ready") - if ready == nil || ready.Status != metav1.ConditionFalse { - t.Fatalf("Ready condition = %+v, want Status=False", ready) - } - if ready.Reason != "ExternalEndpointMissing" { - t.Fatalf("Ready reason = %q, want ExternalEndpointMissing", ready.Reason) - } - // Progressing reason mirrors Ready's reason on the missing path so - // `kubectl describe` shows a coherent pair. - progressing := findCondition(got.Status.Conditions, "Progressing") - if progressing == nil || progressing.Reason != "ExternalEndpointMissing" { - t.Fatalf("Progressing = %+v, want reason ExternalEndpointMissing", progressing) - } -} - -func TestReconcileExternalWhitespaceEndpointTreatedAsMissing(t *testing.T) { - // Admission rejects a whitespace-only spec.remoteStorage.endpoint, but a - // caller that bypasses admission can still construct one. - // The reconciler must treat it as missing — publishing a raw - // "LMCACHE_REMOTE_URL=lm:// " to the engine env is worse than - // publishing nothing, and Ready=True on whitespace would mislead - // every downstream consumer. - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-ws", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(" \t "), - }, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, "ext-ws", "default") - - got := getBackend(t, r, "ext-ws", "default") - if got.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want empty (whitespace must be trimmed)", got.Status.Endpoint) - } - ready := findCondition(got.Status.Conditions, "Ready") - if ready == nil || ready.Status != metav1.ConditionFalse { - t.Fatalf("Ready = %+v, want Status=False", ready) - } - if ready.Reason != "ExternalEndpointMissing" { - t.Fatalf("Ready reason = %q, want ExternalEndpointMissing", ready.Reason) - } -} - -func TestReconcileExternalClearsRemovedEndpoint(t *testing.T) { - scheme := newScheme(t) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "default"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(""), - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: "stale-cache.default.svc:8080"}, - } - r := newReconciler(scheme, cb) - - reconcile(t, r, "example", "default") - - if got := getBackend(t, r, "example", "default").Status.Endpoint; got != "" { - t.Fatalf("status.endpoint = %q, want empty", got) - } -} diff --git a/internal/controller/cachebackend_status.go b/internal/controller/cachebackend_status.go index f28802a2..dc3361bf 100644 --- a/internal/controller/cachebackend_status.go +++ b/internal/controller/cachebackend_status.go @@ -140,9 +140,8 @@ const ( // no port, embedded whitespace, unbracketed IPv6, …). Current admission // rejects all of these; this defensive reason covers objects that bypassed // admission. Status reflects the gap loudly rather than advertising the - // malformed value as Ready=True (which would let the pod webhook then inject - // an LMCACHE_REMOTE_URL the engine connector refuses at startup — turning a - // cache misconfiguration into a serving outage). + // malformed value as Ready=True (which would let the pod webhook inject a + // remote adapter target the engine rejects at startup). conditionReasonExternalEndpointInvalid = "ExternalEndpointInvalid" ) @@ -228,7 +227,6 @@ func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backen engineCompatMsg, engineCompatObserved := r.detectEngineConnectorCrashLoop(ctx, backend) prevEngineIncompatible := meta.IsStatusConditionFalse(backend.Status.Conditions, conditionTypeEngineCompatibility) err := r.patchStatus(ctx, backend, func() { - backend.Status.Endpoint = endpoint setRemoteStorageStatus(backend, endpoint, remoteStatus, remoteReason, remoteMessage, publishedGen) backend.Status.ObservedGeneration = publishedGen // Latch the first KV-event observation write-once. The poller can later @@ -608,13 +606,6 @@ const ( // to have observed its current generation and to have enough updated + // available replicas, so a stale rollout (e.g. mid image change) is never // reported Ready. -// -// When the CacheBackend is autoscaled the HPA owns the desired replica count, -// so the comparison target is the live Deployment's spec.replicas (which the -// HPA writes) rather than the CacheBackend's spec.replicas (which is ignored -// in that mode). This keeps Ready accurate when an HPA decides to run more -// pods than spec.replicas, and avoids a false ScaledToZero when spec.replicas -// happens to be 0 with autoscaling configured. func managedReadiness(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) (metav1.ConditionStatus, string, string) { want := desiredReplicas(backend, dep) @@ -659,37 +650,12 @@ func progressingFromReady(readyStatus metav1.ConditionStatus, reason, message st } } -// desiredReplicas is the per-reconcile source of truth for "how many replicas -// should this backend be running". With autoscaling enabled the HPA writes -// spec.replicas on the Deployment, so the live value is authoritative; without -// it, the user's spec.replicas (default 1) wins. -// -// It applies the same singleton clamp the render path does (clampSingletonReplicas): -// readiness must expect the count actually DEPLOYED, not the CR's grandfathered -// spec.replicas. Without this, a singleton backend (SGLang Redis L2, or a -// host-network Mooncake master) whose spec.replicas was set to 3 before admission -// rejected it deploys one pod but expects three, and reports RolloutInProgress -// forever. spec.replicas 0 (disabled) is preserved. -func desiredReplicas(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) int32 { - want := unclampedDesiredReplicas(backend, dep) - if want > 1 && cacheServerIsSingleton(backend, &dep.Spec.Template.Spec) { - return 1 - } - return want -} - -func unclampedDesiredReplicas(backend *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) int32 { - if backend.Spec.Autoscaling != nil { - // First reconcile after an HPA spec is added may briefly see - // dep.Spec.Replicas still set by the controller; the HPA will overwrite - // it within one cycle. Until then, fall back to the controller value. - if dep.Spec.Replicas != nil { - return *dep.Spec.Replicas - } - // Fall through to the floor. - } - if backend.Spec.Replicas != nil { - return *backend.Spec.Replicas +// desiredReplicas reads the live managed Redis Deployment. The controller +// renders one replica, and the live value keeps readiness aligned with the +// workload actually observed. +func desiredReplicas(_ *cachev1alpha1.CacheBackend, dep *appsv1.Deployment) int32 { + if dep.Spec.Replicas != nil { + return *dep.Spec.Replicas } return 1 } diff --git a/internal/controller/cachebackend_status_test.go b/internal/controller/cachebackend_status_test.go index deeb9762..8b2d4e08 100644 --- a/internal/controller/cachebackend_status_test.go +++ b/internal/controller/cachebackend_status_test.go @@ -22,10 +22,9 @@ import ( "time" ) -func TestReconcileLMCacheReadyWhenReplicasAvailable(t *testing.T) { +func TestReconcileManagedRedisReadyWhenReplicaAvailable(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) r := newReconciler(scheme, cb) reconcile(t, r, "cache", "ns1") @@ -42,14 +41,13 @@ func TestReconcileLMCacheReadyWhenReplicasAvailable(t *testing.T) { reconcile(t, r, "cache", "ns1") updated := getBackend(t, r, "cache", "ns1") - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionTrue { - t.Fatalf("Ready condition = %+v, want True", cond) + if cond := findCondition(updated.Status.Conditions, conditionTypeRemoteStorageReady); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("RemoteStorageReady = %+v, want True", cond) } } func TestManagedReadinessGatesReadyOnRollout(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(2) cases := []struct { name string @@ -58,16 +56,17 @@ func TestManagedReadinessGatesReadyOnRollout(t *testing.T) { wantReason string }{ { - name: "fresh create, nothing ready", - dep: appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}}, + name: "fresh create, nothing ready", + dep: appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: ptrInt32(2)}}, wantStatus: metav1.ConditionFalse, wantReason: conditionReasonRolloutInProgress, }, { name: "stale rollout after image change (old pods still available)", dep: appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Generation: 2}, - Status: appsv1.DeploymentStatus{ObservedGeneration: 1, UpdatedReplicas: 0, AvailableReplicas: 2, ReadyReplicas: 2}, + ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: ptrInt32(2)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1, UpdatedReplicas: 0, AvailableReplicas: 2, ReadyReplicas: 2}, }, wantStatus: metav1.ConditionFalse, wantReason: conditionReasonRolloutInProgress, @@ -75,8 +74,8 @@ func TestManagedReadinessGatesReadyOnRollout(t *testing.T) { { name: "rolled out and available", dep: appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Generation: 2}, - Status: appsv1.DeploymentStatus{ObservedGeneration: 2, UpdatedReplicas: 2, AvailableReplicas: 2, ReadyReplicas: 2}, + ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: ptrInt32(2)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 2, UpdatedReplicas: 2, AvailableReplicas: 2, ReadyReplicas: 2}, }, wantStatus: metav1.ConditionTrue, wantReason: conditionReasonBackendReady, @@ -84,8 +83,8 @@ func TestManagedReadinessGatesReadyOnRollout(t *testing.T) { { name: "rolled out but replicas unavailable", dep: appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Generation: 2}, - Status: appsv1.DeploymentStatus{ObservedGeneration: 2, UpdatedReplicas: 2, AvailableReplicas: 1, ReadyReplicas: 1}, + ObjectMeta: metav1.ObjectMeta{Generation: 2}, Spec: appsv1.DeploymentSpec{Replicas: ptrInt32(2)}, + Status: appsv1.DeploymentStatus{ObservedGeneration: 2, UpdatedReplicas: 2, AvailableReplicas: 1, ReadyReplicas: 1}, }, wantStatus: metav1.ConditionFalse, wantReason: conditionReasonReplicasUnavailable, @@ -103,10 +102,10 @@ func TestManagedReadinessGatesReadyOnRollout(t *testing.T) { func TestManagedReadinessZeroReplicasNotReady(t *testing.T) { cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(0) // Even a fully-observed Deployment with 0/0 replicas must not be Ready. dep := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Generation: 1}, + Spec: appsv1.DeploymentSpec{Replicas: ptrInt32(0)}, Status: appsv1.DeploymentStatus{ObservedGeneration: 1}, } if status, reason, _ := managedReadiness(cb, &dep); status == metav1.ConditionTrue { @@ -132,13 +131,13 @@ func TestReconcileLifecycleExitsClearProbeRateLimiter(t *testing.T) { mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = externalLMCacheStorage("external.ns1.svc:8080") + cb.Spec.RemoteStorage = externalRedisStorage("external.ns1.svc:6379") }, }, { - name: "managed → Unmanaged (StatefulSet kind)", + name: "managed → Unmanaged (unsupported type)", mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") }, }, } @@ -193,13 +192,13 @@ func TestReconcileLifecycleExitsClearEngineCompatibility(t *testing.T) { mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = externalLMCacheStorage("external.ns1.svc:8080") + cb.Spec.RemoteStorage = externalRedisStorage("external.ns1.svc:6379") }, }, { - name: "managed → Unmanaged (StatefulSet kind)", + name: "managed → Unmanaged (unsupported type)", mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet + cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") }, }, } @@ -316,7 +315,6 @@ func TestUpdateManagedStatusPreservesEngineCompatibilityOnListError(t *testing.T func TestReconcileLMCacheStatusIndependentOfApplyError(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) var blockDeploymentUpdate atomic.Bool gr := schema.GroupResource{Group: "apps", Resource: "deployments"} @@ -342,7 +340,7 @@ func TestReconcileLMCacheStatusIndependentOfApplyError(t *testing.T) { // an Update to happen by changing the image in the CR. blockDeploymentUpdate.Store(true) live := getBackend(t, r, "cache", "ns1") - live.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v9" + live.Spec.RemoteStorage.Redis.Image = "example.com/redis:v9" live.Generation = 2 if err := r.Update(context.Background(), live); err != nil { t.Fatalf("update CR: %v", err) @@ -357,8 +355,8 @@ func TestReconcileLMCacheStatusIndependentOfApplyError(t *testing.T) { } updated := getBackend(t, r, "cache", "ns1") - if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionTrue { - t.Fatalf("Ready condition = %+v, want True (status must reflect live Deployment regardless of apply error)", cond) + if cond := findCondition(updated.Status.Conditions, conditionTypeRemoteStorageReady); cond == nil || cond.Status != metav1.ConditionTrue { + t.Fatalf("RemoteStorageReady = %+v, want True (status must reflect live Deployment regardless of apply error)", cond) } // Apply for generation 2 failed, so observedGeneration must NOT have // advanced to 2 — it should still report 1 (the last generation we @@ -371,7 +369,7 @@ func TestReconcileLMCacheStatusIndependentOfApplyError(t *testing.T) { if cond := findCondition(updated.Status.Conditions, conditionTypeReady); cond == nil || cond.ObservedGeneration != 1 { t.Fatalf("Ready condition ObservedGeneration = %d, want 1", cond.ObservedGeneration) } - if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got == "example.com/lmcache-server:v9" { + if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got == "example.com/redis:v9" { t.Fatalf("deployment image was updated despite Forbidden — interceptor was not exercised") } } diff --git a/internal/controller/cachebackend_t2degraded_test.go b/internal/controller/cachebackend_t2degraded_test.go index 078fbbd8..757836a4 100644 --- a/internal/controller/cachebackend_t2degraded_test.go +++ b/internal/controller/cachebackend_t2degraded_test.go @@ -114,7 +114,7 @@ func TestIntegrationT2DegradedCondition(t *testing.T) { } cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = externalLMCacheStorage("shared.svc.cluster.local:9000") + cb.Spec.RemoteStorage = externalRedisStorage("shared.svc.cluster.local:6379") if err := k8s.Update(ctx, &cb); err != nil { t.Fatalf("flip to External: %v", err) } diff --git a/internal/controller/cachebackend_workload.go b/internal/controller/cachebackend_workload.go index 303c5afc..9b45073d 100644 --- a/internal/controller/cachebackend_workload.go +++ b/internal/controller/cachebackend_workload.go @@ -8,35 +8,25 @@ import ( "context" "fmt" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/controller-runtime/pkg/log" -) - -// Default HPA tuning when the autoscaling spec leaves them unset. -const ( - defaultHPAMinReplicas = int32(1) - defaultHPATargetCPUUtilizationPercent = int32(80) ) // buildDeployment wraps the adapter-rendered PodSpec into a Deployment the // controller owns: ObjectMeta + labels + replicas + selector come from the // CacheBackend identity, not the adapter. func (r *CacheBackendReconciler) buildDeployment(backend *cachev1alpha1.CacheBackend, podSpec *corev1.PodSpec) *appsv1.Deployment { - replicas := initialReplicas(backend) + replicas := int32(1) selector := selectorLabels(backend.Name) podLabels := podTemplateLabels(backend) pod := podSpec.DeepCopy() - applyPodOverrides(pod, backend.Spec.Template) + applyManagedWorkload(pod, backend.Spec.RemoteStorage) dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -54,73 +44,9 @@ func (r *CacheBackendReconciler) buildDeployment(backend *cachev1alpha1.CacheBac }, } - // A hostNetwork cache-server binds its ports directly on the node, so the - // default RollingUpdate would surge a second pod onto the same host ports. - // That surge pod cannot serve: it CrashLoops failing to bind while the old pod - // still holds the port. In practice the scheduler rejects it even earlier, - // because the API server defaults hostPort=containerPort for hostNetwork pods - // (core/v1 defaultHostNetworkPorts — confirmed against a live apiserver: a - // hostNetwork pod declaring only containerPort=50051 comes back with - // hostPort=50051), which trips the NodePorts predicate. Recreate tears the old - // pod down first, so neither failure mode is reachable. - // Only a backend whose data plane requires the host network renders one - // (Mooncake today); every other adapter keeps the default RollingUpdate. - if pod.HostNetwork { - dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} - } - clampSingletonReplicas(&dep.Spec, backend) return dep } -// cacheServerIsSingleton reports whether the backend's managed cache-server must run -// as exactly one replica — no scale-out, no HPA. Two backends require it, for -// different reasons: -// -// - a host-network server (the Mooncake master): a second replica cannot bind the -// node ports the first already holds, and on a different node comes up as an -// independent master that silently splits the store; -// - the (sglang, LMCache) Redis L2 store: a plain Redis is not clustered, so a -// second pod behind the one Service shards the keyspace across independent -// instances and silently partitions the cache. -// -// Admission rejects spec.replicas>1 / spec.autoscaling for both -// (rejectMooncakeMasterScaleOut, rejectSGLangRedisL2ScaleOut); this is the shared -// predicate the reconciler backstop keys on. -func cacheServerIsSingleton(backend *cachev1alpha1.CacheBackend, pod *corev1.PodSpec) bool { - // EventsOnly renders no cache-server at all (the reconciler sheds any owned - // workload), so there is no singleton to protect — keep the predicate honest, and - // aligned with the admission rules, which exempt EventsOnly for the same reason. - if backend.Spec.IsEventsOnly() { - return false - } - if pod != nil && pod.HostNetwork { - return true - } - return adapterruntime.ResolveRuntimeID(backend) == adapterruntime.RuntimeSGLang && - backend.Spec.Type == cachev1alpha1.CacheBackendTypeLMCache -} - -// clampSingletonReplicas caps a singleton cache-server (see cacheServerIsSingleton) -// at one replica. -// -// Admission rejects spec.replicas>1 and spec.autoscaling for such a backend, but -// that is not sufficient: ValidateUpdate only rejects violations an edit -// *introduces*, so an object written before the rule existed — or before its -// backend moved onto a singleton data plane — stays in etcd with replicas=3 and an -// HPA, and is never re-validated. Rendering that faithfully would schedule several -// servers: host-network masters contend for the same node ports or split the store; -// several Redis pods behind one Service partition the keyspace. -// -// The reconciler is therefore the last line of defense, and it clamps rather than -// obeys. 0 is preserved: that is "disabled", not "scaled out". -func clampSingletonReplicas(spec *appsv1.DeploymentSpec, backend *cachev1alpha1.CacheBackend) { - if !cacheServerIsSingleton(backend, &spec.Template.Spec) || spec.Replicas == nil || *spec.Replicas <= 1 { - return - } - one := int32(1) - spec.Replicas = &one -} - // buildService wraps the adapter-rendered Service spec into a Service the // controller owns: ObjectMeta + Selector come from the CacheBackend identity. // Adapter-provided fields (Spec.Type, Spec.Ports) are preserved as-is. @@ -160,12 +86,12 @@ func podTemplateLabels(backend *cachev1alpha1.CacheBackend) map[string]string { return labels } -// applyPodOverrides copies optional pod-level scheduling/security overrides -// from the spec onto the rendered pod spec. Server-defaulted fields -// (schedulerName, terminationGracePeriodSeconds) are always set to their -// defaults when unset so the rendered template matches the API-server- -// defaulted object and updates don't churn. -func applyPodOverrides(spec *corev1.PodSpec, override *cachev1alpha1.CacheBackendPodSpecOverride) { +// applyManagedWorkload materializes server-defaulted fields and applies the +// scheduling/security contract for a controller-managed remote provider. +// schedulerName and terminationGracePeriodSeconds are always materialized so +// the desired template matches the API-server-defaulted object and updates do +// not churn. +func applyManagedWorkload(spec *corev1.PodSpec, storage *cachev1alpha1.CacheBackendRemoteStorageSpec) { if spec.SchedulerName == "" { spec.SchedulerName = "default-scheduler" } @@ -173,18 +99,37 @@ func applyPodOverrides(spec *corev1.PodSpec, override *cachev1alpha1.CacheBacken defaultGrace := int64(30) spec.TerminationGracePeriodSeconds = &defaultGrace } - if override == nil { + if storage == nil || storage.Ownership != cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged || storage.Workload == nil { return } - spec.NodeSelector = override.NodeSelector - spec.Affinity = override.Affinity - spec.Tolerations = override.Tolerations - spec.TopologySpreadConstraints = override.TopologySpreadConstraints - spec.ImagePullSecrets = override.ImagePullSecrets - spec.ServiceAccountName = override.ServiceAccountName - spec.SecurityContext = override.SecurityContext - spec.PriorityClassName = override.PriorityClassName - spec.RuntimeClassName = override.RuntimeClassName + override := storage.Workload.DeepCopy() + if override.NodeSelector != nil { + spec.NodeSelector = override.NodeSelector + } + if override.Affinity != nil { + spec.Affinity = override.Affinity + } + if override.Tolerations != nil { + spec.Tolerations = override.Tolerations + } + if override.TopologySpreadConstraints != nil { + spec.TopologySpreadConstraints = override.TopologySpreadConstraints + } + if override.ImagePullSecrets != nil { + spec.ImagePullSecrets = override.ImagePullSecrets + } + if override.ServiceAccountName != "" { + spec.ServiceAccountName = override.ServiceAccountName + } + if override.SecurityContext != nil { + spec.SecurityContext = override.SecurityContext + } + if override.PriorityClassName != "" { + spec.PriorityClassName = override.PriorityClassName + } + if override.RuntimeClassName != nil { + spec.RuntimeClassName = override.RuntimeClassName + } if override.SchedulerName != "" { spec.SchedulerName = override.SchedulerName } @@ -193,9 +138,8 @@ func applyPodOverrides(spec *corev1.PodSpec, override *cachev1alpha1.CacheBacken } } -// serviceEndpoint formats the published cache endpoint as host:port using the -// service's first port. Engine-protocol prefixes (e.g. lm:// for LMCache) are -// the adapter's responsibility — status.endpoint stays engine-agnostic. +// serviceEndpoint formats the managed remote-storage endpoint as host:port +// using the Service's first port. func serviceEndpoint(svc *corev1.Service) string { if len(svc.Spec.Ports) == 0 { return "" @@ -208,20 +152,12 @@ func serviceEndpoint(svc *corev1.Service) string { // On create we establish the full templated spec. On update we touch only the // fields this module owns — replicas, the rollout strategy, and everything // reconcileManagedPodSpec covers (the managed container's image/command/args/env, -// pod-level overrides and volumes, HostNetwork/DNSPolicy) — and leave everything +// pod-level defaults and volumes) — and leave everything // else intact. Overwriting the whole PodTemplate would strip API-server-defaulted // fields (port Protocol, RestartPolicy, probe thresholds, ...), and since those are // re-defaulted on every write it would spin a perpetual update loop via the // Owns(Deployment) watch. // -// When an HPA owns scaling (spec.autoscaling set), the reconciler defers to the -// HPA's replica count rather than overwriting it — re-asserting replicas on -// every reconcile would fight the HPA and churn the rollout. The one exception is -// a singleton cache-server (a host-network master, or the SGLang Redis L2 — see -// cacheServerIsSingleton): clampSingletonReplicas runs last and overrides both the -// spec and the HPA (see its godoc for why admission alone cannot protect a -// grandfathered object). -// // Wrapped in RetryOnConflict because the kube Deployment controller writes // Deployment.Status often during rollout; without retry, the Get/Update inside // CreateOrUpdate races those writes and surfaces a 409 that aborts the @@ -230,11 +166,6 @@ func (r *CacheBackendReconciler) applyDeployment(ctx context.Context, backend *c err := retry.RetryOnConflict(retry.DefaultRetry, func() error { dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} _, err := controllerutil.CreateOrUpdate(ctx, r.Client, dep, func() error { - // Snapshot the HPA-owned field BEFORE we mutate the live spec. When an - // HPA is configured the controller must never re-assert replicas; doing - // so would fight the HPA on every reconcile and churn the rollout. - liveReplicas := dep.Spec.Replicas - dep.Labels = desired.Labels if dep.CreationTimestamp.IsZero() { dep.Spec = *desired.Spec.DeepCopy() @@ -259,20 +190,6 @@ func (r *CacheBackendReconciler) applyDeployment(ctx context.Context, backend *c dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: wantStrategy} } } - if backend.Spec.Autoscaling != nil && liveReplicas != nil { - // Preserve the HPA's writes — but clamp to the configured floor so - // raising autoscaling.minReplicas doesn't briefly publish Ready - // against the old smaller live count before the HPA catches up. - preserved := *liveReplicas - if floor := autoscalingFloor(backend.Spec.Autoscaling); preserved < floor { - preserved = floor - } - dep.Spec.Replicas = &preserved - } - // LAST, after every other writer (desired spec, HPA preservation): a - // singleton cache-server must never be scaled out, no matter what the - // spec or a stale HPA asks for. - clampSingletonReplicas(&dep.Spec, backend) return controllerutil.SetControllerReference(backend, dep, r.Scheme) }) return err @@ -283,59 +200,12 @@ func (r *CacheBackendReconciler) applyDeployment(ctx context.Context, backend *c return nil } -// autoscalingFloor is the effective minReplicas value for the HPA — the -// user's setting, or the default floor when unset. Mirrors the resolution -// buildHPA does so the reconciler and the HPA agree on the lower bound. -func autoscalingFloor(spec *cachev1alpha1.CacheBackendAutoscalingSpec) int32 { - if spec == nil { - return defaultHPAMinReplicas - } - if spec.MinReplicas != nil { - return *spec.MinReplicas - } - return defaultHPAMinReplicas -} - // applyService creates or updates the backend Service idempotently, owned by the CR. // Type, selector, and ports are reconciled (so out-of-band drift is corrected); the // rendered ports carry Protocol=TCP so they match the API-server-defaulted object, // and nodePort stays an allocated field we never touch — so reconciling ports does // not churn through the Owns(Service) watch. -// -// clusterIP is the exception. It is IMMUTABLE after creation and it is the field -// that decides whether the Service is headless. A backend whose data plane is a -// peer-to-peer mesh (Mooncake) must be headless, so the Service DNS name resolves -// to the pod's (hostNetwork: node) IP with every dynamically negotiated port -// reachable; a virtual ClusterIP forwards only the declared ports and strands the -// mesh. Therefore: -// - on create we propagate the adapter's clusterIP (e.g. "None"), -// - on update we never touch it (immutable), -// - and when a live Service's headless-ness diverges from what the adapter now -// renders, we delete it so the next reconcile recreates it correctly. Without -// that, the in-place update would fail forever and the backend would stay -// silently broken — Ready, but transferring nothing. func (r *CacheBackendReconciler) applyService(ctx context.Context, backend *cachev1alpha1.CacheBackend, desired *corev1.Service) error { - var live corev1.Service - switch err := r.Get(ctx, client.ObjectKeyFromObject(desired), &live); { - case err == nil: - // Only ever delete a Service this CacheBackend controls. - if metav1.IsControlledBy(&live, backend) && - headlessnessDiverges(live.Spec.ClusterIP, desired.Spec.ClusterIP) { - if delErr := r.Delete(ctx, &live); delErr != nil && !apierrors.IsNotFound(delErr) { - return fmt.Errorf("recreate service %s/%s for immutable clusterIP change: %w", - desired.Namespace, desired.Name, delErr) - } - log.FromContext(ctx).Info("deleted service to change immutable clusterIP; recreating", - "namespace", desired.Namespace, "name", desired.Name, - "liveClusterIP", live.Spec.ClusterIP, "desiredClusterIP", desired.Spec.ClusterIP) - // The Owns(Service) watch requeues on this delete; recreate on the next - // pass rather than racing a stale cache read inside CreateOrUpdate below. - return nil - } - case !apierrors.IsNotFound(err): - return fmt.Errorf("get service %s/%s: %w", desired.Namespace, desired.Name, err) - } - err := retry.RetryOnConflict(retry.DefaultRetry, func() error { svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} _, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { @@ -343,8 +213,8 @@ func (r *CacheBackendReconciler) applyService(ctx context.Context, backend *cach svc.Spec.Type = desired.Spec.Type svc.Spec.Selector = desired.Spec.Selector svc.Spec.Ports = desired.Spec.Ports - // Settable only at creation. On an existing Service this is already - // "None" or an allocated VIP, and a divergent one was deleted above. + // ClusterIP is settable only at creation. Managed Redis uses the normal + // allocated ClusterIP when the renderer leaves this empty. if svc.Spec.ClusterIP == "" { svc.Spec.ClusterIP = desired.Spec.ClusterIP } @@ -358,20 +228,6 @@ func (r *CacheBackendReconciler) applyService(ctx context.Context, backend *cach return nil } -// headlessnessDiverges reports whether a live Service's clusterIP allocation is -// incompatible with what the adapter now renders. spec.clusterIP is immutable, so -// a Service cannot be migrated in place between headless ("None") and a virtual -// ClusterIP in either direction — it must be recreated. -// -// An empty live value means the API server has not assigned one yet; treat that as -// "no divergence" so a transient read never triggers a delete. -func headlessnessDiverges(live, desired string) bool { - if live == "" { - return false - } - return (live == corev1.ClusterIPNone) != (desired == corev1.ClusterIPNone) -} - // reconcileManagedPodSpec updates the spec-driven fields of the live pod spec in // place: the managed container's image/command/args/env, the pod-level override // fields, and the networking the adapter owns (HostNetwork, and DNSPolicy @@ -411,11 +267,7 @@ func reconcileManagedPodSpec(live *corev1.PodSpec, desired *corev1.PodSpec) { live.RuntimeClassName = desired.RuntimeClassName live.TerminationGracePeriodSeconds = desired.TerminationGracePeriodSeconds - // Host networking is part of the adapter's desired shape, not an - // API-server-defaulted field: a backend whose data plane is a peer-to-peer - // mesh (Mooncake) is unreachable on overlay pod IPs. Reconcile it on UPDATE - // too, or an already-provisioned backend would never migrate onto the host - // network and would keep transferring nothing. + // Keep generic PodSpec network fields aligned with the provider renderer. live.HostNetwork = desired.HostNetwork // dnsPolicy IS API-server-defaulted (ClusterFirst), but it must still be // reconciled in BOTH directions: a hostNetwork backend needs @@ -433,14 +285,9 @@ func reconcileManagedPodSpec(live *corev1.PodSpec, desired *corev1.PodSpec) { // reconcileManagedContainer updates the spec-driven fields of the managed backend // container in place, leaving API-server-defaulted container fields untouched. // -// Containers in live whose names are not in desired are dropped — this is the -// upgrade path from a previous colocated all-in-one rendering (container -// name "vllm") to the standalone topology (container name "lmcache-server"): -// an in-place upgrade must replace the old managed container, not stack the -// new one alongside it. We never drop containers that match a desired name -// (we only update their managed fields), so a Deployment carrying sidecars -// in addition to the managed container loses the sidecars — sidecars were -// not supported in the previous rendering and remain unsupported here. +// Containers in live whose names are not in desired are dropped. The managed +// provider Deployment is fully controller-owned, including its container set; +// operator-added sidecars are not supported and must not survive reconciliation. func reconcileManagedContainer(live *corev1.PodSpec, desired *corev1.PodSpec) { if len(desired.Containers) == 0 { return @@ -450,8 +297,7 @@ func reconcileManagedContainer(live *corev1.PodSpec, desired *corev1.PodSpec) { desiredNames[desired.Containers[i].Name] = i } - // First pass: drop any live container whose name isn't desired (the - // upgrade-from-previous-managed-shape case). + // First pass: drop any live container whose name isn't desired. kept := live.Containers[:0] for i := range live.Containers { if _, ok := desiredNames[live.Containers[i].Name]; ok { @@ -499,11 +345,8 @@ func reconcileManagedContainer(live *corev1.PodSpec, desired *corev1.PodSpec) { } } -// cleanupOwnedWorkload best-effort deletes the Deployment + Service + HPA this -// CR owns, used when a backend is no longer a managed Deployment (type/kind -// changed). Normal CR deletion is handled by owner-reference garbage -// collection; this covers the in-place mutation case where the CR itself -// still exists. +// cleanupOwnedWorkload best-effort deletes the Deployment and Service this CR +// owns when the backend no longer requests managed remote storage. func (r *CacheBackendReconciler) cleanupOwnedWorkload(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} @@ -515,8 +358,7 @@ func (r *CacheBackendReconciler) cleanupOwnedWorkload(ctx context.Context, backe if err := r.deleteIfOwned(ctx, key, &svc, backend); err != nil { return err } - var hpa autoscalingv2.HorizontalPodAutoscaler - return r.deleteIfOwned(ctx, key, &hpa, backend) + return nil } // deleteIfOwned deletes obj only if it exists and is controller-owned by backend. @@ -532,112 +374,3 @@ func (r *CacheBackendReconciler) deleteIfOwned(ctx context.Context, key types.Na } return nil } - -// initialReplicas picks the Deployment's initial replica count. With -// autoscaling configured, spec.autoscaling.minReplicas is the source of truth -// (defaulting to 1 when unset), so the workload comes up at or above the HPA -// floor on first apply instead of starting at 1 and waiting for the HPA to -// patch it. Without autoscaling, spec.replicas wins (default 1). -func initialReplicas(backend *cachev1alpha1.CacheBackend) int32 { - if backend.Spec.Autoscaling != nil { - if backend.Spec.Autoscaling.MinReplicas != nil { - return *backend.Spec.Autoscaling.MinReplicas - } - return 1 - } - if backend.Spec.Replicas != nil { - return *backend.Spec.Replicas - } - return 1 -} - -// reconcileHPA creates, updates, or deletes the HorizontalPodAutoscaler that -// drives the backend Deployment's replica count. The HPA exists iff -// spec.autoscaling is set; otherwise any controller-owned HPA is removed. -func (r *CacheBackendReconciler) reconcileHPA(ctx context.Context, backend *cachev1alpha1.CacheBackend, deployment *appsv1.Deployment) error { - // A singleton cache-server (see cacheServerIsSingleton): an HPA would fight the - // clampSingletonReplicas clamp on every reconcile and, whenever it won, put a - // second server on the cluster — a split store (host-network master) or a - // partitioned keyspace (Redis L2). Admission rejects spec.autoscaling for these - // backends, but a grandfathered object still carries one — so tear the HPA down - // from the observed shape rather than trusting the spec. - if backend.Spec.Autoscaling == nil || cacheServerIsSingleton(backend, &deployment.Spec.Template.Spec) { - // Autoscaling disabled (or impossible) — clean up any HPA we previously owned. - return r.deleteOwnedHPA(ctx, backend, deployment.Name) - } - - desired := buildHPA(backend, deployment) - hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: metav1.ObjectMeta{Name: desired.Name, Namespace: desired.Namespace}} - _, err := controllerutil.CreateOrUpdate(ctx, r.Client, hpa, func() error { - hpa.Labels = desired.Labels - hpa.Spec = desired.Spec - return controllerutil.SetControllerReference(backend, hpa, r.Scheme) - }) - if err != nil { - return fmt.Errorf("apply HPA %s/%s: %w", desired.Namespace, desired.Name, err) - } - return nil -} - -// buildHPA renders the desired HorizontalPodAutoscaler for a CacheBackend whose -// spec.autoscaling is set. Targets the managed Deployment by name. Phase 1 ships -// a CPU-utilization target; cache-aware (custom-metric) HPAs come later. -func buildHPA(backend *cachev1alpha1.CacheBackend, deployment *appsv1.Deployment) *autoscalingv2.HorizontalPodAutoscaler { - spec := backend.Spec.Autoscaling - minReplicas := defaultHPAMinReplicas - if spec.MinReplicas != nil { - minReplicas = *spec.MinReplicas - } - target := defaultHPATargetCPUUtilizationPercent - if spec.TargetCPUUtilizationPercent != nil { - target = *spec.TargetCPUUtilizationPercent - } - return &autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{ - Name: deployment.Name, - Namespace: deployment.Namespace, - Labels: deployment.Labels, - }, - Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ - APIVersion: "apps/v1", - Kind: "Deployment", - Name: deployment.Name, - }, - MinReplicas: &minReplicas, - MaxReplicas: spec.MaxReplicas, - Metrics: []autoscalingv2.MetricSpec{ - { - Type: autoscalingv2.ResourceMetricSourceType, - Resource: &autoscalingv2.ResourceMetricSource{ - Name: corev1.ResourceCPU, - Target: autoscalingv2.MetricTarget{ - Type: autoscalingv2.UtilizationMetricType, - AverageUtilization: &target, - }, - }, - }, - }, - }, - } -} - -// deleteOwnedHPA removes a previously-owned HPA (e.g. spec.autoscaling cleared). -// Missing HPA is a no-op. -func (r *CacheBackendReconciler) deleteOwnedHPA(ctx context.Context, backend *cachev1alpha1.CacheBackend, name string) error { - key := types.NamespacedName{Name: name, Namespace: backend.Namespace} - var hpa autoscalingv2.HorizontalPodAutoscaler - if err := r.Get(ctx, key, &hpa); err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return fmt.Errorf("get HPA %s/%s: %w", backend.Namespace, name, err) - } - if !metav1.IsControlledBy(&hpa, backend) { - return nil - } - if err := r.Delete(ctx, &hpa); err != nil { - return client.IgnoreNotFound(err) - } - return nil -} diff --git a/internal/controller/cachebackend_workload_test.go b/internal/controller/cachebackend_workload_test.go index 753dd07c..4afbeb34 100644 --- a/internal/controller/cachebackend_workload_test.go +++ b/internal/controller/cachebackend_workload_test.go @@ -13,56 +13,94 @@ import ( "testing" ) -func TestReconcileLMCacheImageOverride(t *testing.T) { +func TestReconcileManagedWorkloadOverrides(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.RemoteStorage.LMCacheServer.Image = "registry.example.com/lmcache-server:pinned" + grace := int64(45) + runtimeClass := "gvisor" + cb.Spec.RemoteStorage.Workload = &cachev1alpha1.CacheBackendManagedWorkloadSpec{ + NodeSelector: map[string]string{"pool": "cache"}, + Affinity: &corev1.Affinity{}, + Tolerations: []corev1.Toleration{{Key: "cache", Operator: corev1.TolerationOpExists}}, + TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "topology.kubernetes.io/zone", WhenUnsatisfiable: corev1.ScheduleAnyway}}, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "registry-auth"}}, + ServiceAccountName: "cache-provider", + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: func() *bool { v := true; return &v }()}, + PriorityClassName: "cache-critical", + SchedulerName: "cache-scheduler", + RuntimeClassName: &runtimeClass, + TerminationGracePeriodSeconds: &grace, + } r := newReconciler(scheme, cb) reconcile(t, r, "cache", "ns1") - dep := getDeployment(t, r, "cache", "ns1") - if got := dep.Spec.Template.Spec.Containers[0].Image; got != "registry.example.com/lmcache-server:pinned" { - t.Fatalf("container image = %q, want overridden image", got) + pod := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec + if pod.NodeSelector["pool"] != "cache" || pod.Affinity == nil || len(pod.Tolerations) != 1 || len(pod.TopologySpreadConstraints) != 1 { + t.Fatalf("managed workload scheduling not applied: %+v", pod) + } + if pod.ServiceAccountName != "cache-provider" || pod.SchedulerName != "cache-scheduler" || pod.PriorityClassName != "cache-critical" { + t.Fatalf("managed workload identity/scheduler not applied: %+v", pod) + } + if len(pod.ImagePullSecrets) != 1 || pod.ImagePullSecrets[0].Name != "registry-auth" || + pod.SecurityContext == nil || pod.SecurityContext.RunAsNonRoot == nil || !*pod.SecurityContext.RunAsNonRoot { + t.Fatalf("managed workload pull/security settings not applied: %+v", pod) + } + if pod.RuntimeClassName == nil || *pod.RuntimeClassName != "gvisor" || + pod.TerminationGracePeriodSeconds == nil || *pod.TerminationGracePeriodSeconds != 45 { + t.Fatalf("managed workload runtime/grace settings not applied: %+v", pod) } -} - -func TestReconcileLMCacheUpdatesImage(t *testing.T) { - scheme := newScheme(t) - r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - - reconcile(t, r, "cache", "ns1") + // Removing the block must clear every controller-owned override and restore + // materialized Kubernetes defaults rather than stranding stale placement on + // the existing Deployment. live := getBackend(t, r, "cache", "ns1") - live.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v2" + live.Spec.RemoteStorage.Workload = nil if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("update image: %v", err) + t.Fatalf("remove managed workload overrides: %v", err) } reconcile(t, r, "cache", "ns1") - - if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got != "example.com/lmcache-server:v2" { - t.Fatalf("deployment image = %q, want updated image", got) + pod = getDeployment(t, r, "cache", "ns1").Spec.Template.Spec + if pod.NodeSelector != nil || pod.Affinity != nil || pod.Tolerations != nil || pod.TopologySpreadConstraints != nil || pod.ImagePullSecrets != nil || + pod.ServiceAccountName != "" || pod.SecurityContext != nil || pod.PriorityClassName != "" || + pod.RuntimeClassName != nil { + t.Fatalf("removed managed workload settings survived reconciliation: %+v", pod) + } + if pod.SchedulerName != "default-scheduler" || + pod.TerminationGracePeriodSeconds == nil || *pod.TerminationGracePeriodSeconds != 30 { + t.Fatalf("managed workload defaults not restored after removal: %+v", pod) } } -func TestReconcileLMCacheScalesReplicas(t *testing.T) { +func TestReconcileLMCacheImageOverride(t *testing.T) { scheme := newScheme(t) cb := lmcacheBackend("cache", "ns1") - cb.Spec.Replicas = ptrInt32(1) + cb.Spec.RemoteStorage.Redis.Image = "registry.example.com/redis:pinned" r := newReconciler(scheme, cb) reconcile(t, r, "cache", "ns1") + dep := getDeployment(t, r, "cache", "ns1") + if got := dep.Spec.Template.Spec.Containers[0].Image; got != "registry.example.com/redis:pinned" { + t.Fatalf("container image = %q, want overridden image", got) + } +} + +func TestReconcileLMCacheUpdatesImage(t *testing.T) { + scheme := newScheme(t) + r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) + + reconcile(t, r, "cache", "ns1") + live := getBackend(t, r, "cache", "ns1") - live.Spec.Replicas = ptrInt32(3) + live.Spec.RemoteStorage.Redis.Image = "example.com/redis:v2" if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("update replicas: %v", err) + t.Fatalf("update image: %v", err) } reconcile(t, r, "cache", "ns1") - dep := getDeployment(t, r, "cache", "ns1") - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 3 { - t.Fatalf("deployment replicas = %v, want 3 after scale", dep.Spec.Replicas) + if got := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec.Containers[0].Image; got != "example.com/redis:v2" { + t.Fatalf("deployment image = %q, want updated image", got) } } @@ -72,7 +110,7 @@ func TestReconcileServicePortDriftCorrected(t *testing.T) { reconcile(t, r, "cache", "ns1") - // Drift the owned Service out-of-band: drop two ports. + // Drift the owned Service out-of-band: drop its Redis port. var svc corev1.Service if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, &svc); err != nil { t.Fatalf("get service: %v", err) @@ -86,41 +124,16 @@ func TestReconcileServicePortDriftCorrected(t *testing.T) { if err := r.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: "ns1"}, &svc); err != nil { t.Fatalf("re-get service: %v", err) } - if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 65432 { - t.Fatalf("service ports = %v, want lm:// 65432 restored after drift", svc.Spec.Ports) - } -} - -func TestReconcileLMCacheUpdatesPodOverrides(t *testing.T) { - scheme := newScheme(t) - r := newReconciler(scheme, lmcacheBackend("cache", "ns1")) - - reconcile(t, r, "cache", "ns1") - - live := getBackend(t, r, "cache", "ns1") - live.Spec.Template = &cachev1alpha1.CacheBackendPodSpecOverride{ - NodeSelector: map[string]string{"accelerator": "h100"}, - ServiceAccountName: "backend-sa", - } - if err := r.Update(context.Background(), live); err != nil { - t.Fatalf("update template overrides: %v", err) - } - reconcile(t, r, "cache", "ns1") - - spec := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec - if spec.NodeSelector["accelerator"] != "h100" { - t.Fatalf("nodeSelector not reconciled: %v", spec.NodeSelector) - } - if spec.ServiceAccountName != "backend-sa" { - t.Fatalf("serviceAccountName = %q, want backend-sa", spec.ServiceAccountName) + if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 6379 { + t.Fatalf("service ports = %v, want Redis 6379 restored after drift", svc.Spec.Ports) } } func TestReconcileLMCacheUpgradeFromColocatedAllInOne(t *testing.T) { // Upgrading an existing Deployment that the retired colocated all-in-one // builder created (single container named "vllm" referencing pod-level - // volumes "cache-home" + "shm") to the standalone shape (single - // container named "lmcache-server", no pod-level volumes) must REPLACE + // volumes "cache-home" + "shm") to the managed Redis shape (single + // container named "redis-l2", no pod-level volumes) must REPLACE // both the container set AND the dangling adapter-owned volumes. Leaving // the old volumes would carry stale config from the previous shape // forever. @@ -151,8 +164,8 @@ func TestReconcileLMCacheUpgradeFromColocatedAllInOne(t *testing.T) { reconcile(t, r, "cache", "ns1") pod := getDeployment(t, r, "cache", "ns1").Spec.Template.Spec - if len(pod.Containers) != 1 || pod.Containers[0].Name != "lmcache-server" { - t.Fatalf("containers = %v, want exactly 1 lmcache-server after upgrade", containerNames(pod.Containers)) + if len(pod.Containers) != 1 || pod.Containers[0].Name != "redis-l2" { + t.Fatalf("containers = %v, want exactly 1 redis-l2 after upgrade", containerNames(pod.Containers)) } for _, v := range pod.Volumes { if v.Name == "cache-home" || v.Name == "shm" { diff --git a/internal/controller/contract_coverage_sweep_test.go b/internal/controller/contract_coverage_sweep_test.go index 054352ec..d76176c1 100644 --- a/internal/controller/contract_coverage_sweep_test.go +++ b/internal/controller/contract_coverage_sweep_test.go @@ -57,7 +57,7 @@ func TestWebhookPollerSelectorFallbackAgreement(t *testing.T) { // Two CacheBackends with identical selectors. "alpha" should win on // both surfaces (lexicographically before "zebra"). The two helpers // produce different shapes — readyCacheBackendForSweep includes - // status.endpoint (the webhook needs it to inject) and cbFixture is + // status.remoteStorage.endpoint (the webhook needs it for Redis) and cbFixture is // selector-only (the poller's attribution doesn't depend on endpoint). cbAlphaWebhook := readyCacheBackendForSweep("alpha", ns, labels) cbZebraWebhook := readyCacheBackendForSweep("zebra", ns, labels) @@ -303,33 +303,19 @@ func TestRefreshSamePodNameAcrossTenantsIsFailSoft(t *testing.T) { // readyCacheBackendForSweep mirrors the unexported readyCacheBackend helper // in internal/webhook/pod (which we can't import across packages). The -// webhook injects only when status.endpoint is populated, so the agreement +// webhook injects only when the managed remote-storage endpoint is populated, so the agreement // test needs a CacheBackend that the webhook will pick up — not the // selector-only cbFixture form the poller tests use. func readyCacheBackendForSweep(name, namespace string, selector map[string]string) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - UID: types.UID("cb-" + namespace + "-" + name + "-uid"), - }, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: selector}, - }, - Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: name + ".cache-ns.svc.cluster.local:65432", - }, - } + backend := lmcacheBackend(name, namespace) + backend.UID = types.UID("cb-" + namespace + "-" + name + "-uid") + backend.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: selector} + backend.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Endpoint: name + ".cache-ns.svc.cluster.local:6379", + Ready: metav1.ConditionTrue, + } + return backend } // runPodWebhookAndCaptureInjectedBy admits an engine pod via the diff --git a/internal/controller/envtest_helpers_test.go b/internal/controller/envtest_helpers_test.go new file mode 100644 index 00000000..a2a7ea42 --- /dev/null +++ b/internal/controller/envtest_helpers_test.go @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) + +func skipWithoutEnvtest(t *testing.T) { + t.Helper() + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS not set; run with `KUBEBUILDER_ASSETS=$(make test-env | tail -1) go test` for envtest") + } + logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stderr))) +} + +func startEnv(t *testing.T) (client.Client, *runtime.Scheme, *rest.Config) { + t.Helper() + env := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + cfg, err := env.Start() + if err != nil { + t.Fatalf("start envtest: %v", err) + } + t.Cleanup(func() { + if err := env.Stop(); err != nil { + t.Logf("stop envtest: %v", err) + } + }) + + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("add client-go scheme: %v", err) + } + if err := cachev1alpha1.AddToScheme(scheme); err != nil { + t.Fatalf("add cache scheme: %v", err) + } + k8s, err := client.New(cfg, client.Options{Scheme: scheme}) + if err != nil { + t.Fatalf("build client: %v", err) + } + return k8s, scheme, cfg +} + +var integrationNamespaceCounter int64 + +func freshNS(t *testing.T, k8s client.Client) string { + t.Helper() + name := fmt.Sprintf("it-%d", atomic.AddInt64(&integrationNamespaceCounter, 1)) + if err := k8s.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}); err != nil { + t.Fatalf("create namespace %s: %v", name, err) + } + return name +} + +func ptrBool(v bool) *bool { return &v } + +func pollDeployment(t *testing.T, k8s client.Client, key client.ObjectKey, what string) string { + t.Helper() + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + var dep appsv1.Deployment + if err := k8s.Get(context.Background(), key, &dep); err == nil { + return string(dep.UID) + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("timed out waiting for deployment to %s", what) + return "" +} diff --git a/internal/controller/integration_test.go b/internal/controller/integration_test.go deleted file mode 100644 index ca0f19cf..00000000 --- a/internal/controller/integration_test.go +++ /dev/null @@ -1,1830 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package controller - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/go-logr/logr" - appsv1 "k8s.io/api/apps/v1" - autoscalingv2 "k8s.io/api/autoscaling/v2" - corev1 "k8s.io/api/core/v1" - eventsv1 "k8s.io/api/events/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/types" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - controlplaneapi "github.com/cachebox-project/inference-cache/internal/controlplaneapi" - podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" -) - -// These tests run the reconciler against a real kube-apiserver (envtest), so they -// cover behavior the fake client cannot: real API-server defaulting and the -// idempotency/no-churn against it, real Status subresource semantics on the -// child Deployment, real CRD validation (e.g. the autoscaling XValidation rule), -// HPA reconciliation, and — in TestIntegrationCacheBackendWatch — the Owns() -// watch re-trigger via a real manager. -// -// Skipped unless KUBEBUILDER_ASSETS is set. CI installs envtest in -// .github/workflows/ci.yml before `make test-race`, so the suite runs there; -// locally run `KUBEBUILDER_ASSETS=$(make test-env | tail -1) go test ./...`. - -func skipWithoutEnvtest(t *testing.T) { - t.Helper() - if os.Getenv("KUBEBUILDER_ASSETS") == "" { - t.Skip("KUBEBUILDER_ASSETS not set; run with `KUBEBUILDER_ASSETS=$(make test-env | tail -1) go test` for envtest") - } - logf.SetLogger(zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stderr))) -} - -// startEnv boots an envtest apiserver with the project CRDs and returns a client. -func startEnv(t *testing.T) (client.Client, *runtime.Scheme, *rest.Config) { - t.Helper() - env := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - cfg, err := env.Start() - if err != nil { - t.Fatalf("start envtest: %v", err) - } - t.Cleanup(func() { - if err := env.Stop(); err != nil { - t.Logf("stop envtest: %v", err) - } - }) - - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatalf("add client-go scheme: %v", err) - } - if err := cachev1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("add cache scheme: %v", err) - } - - k8s, err := client.New(cfg, client.Options{Scheme: scheme}) - if err != nil { - t.Fatalf("build client: %v", err) - } - return k8s, scheme, cfg -} - -var itNSCounter int64 - -func freshNS(t *testing.T, k8s client.Client) string { - t.Helper() - name := fmt.Sprintf("it-%d", atomic.AddInt64(&itNSCounter, 1)) - if err := k8s.Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}); err != nil { - t.Fatalf("create namespace %s: %v", name, err) - } - return name -} - -func getService(t *testing.T, r *CacheBackendReconciler, name, namespace string) *corev1.Service { - t.Helper() - var svc corev1.Service - if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, &svc); err != nil { - t.Fatalf("get service %s/%s: %v", namespace, name, err) - } - return &svc -} - -func getRV(t *testing.T, r *CacheBackendReconciler, name, namespace string, obj client.Object) string { - t.Helper() - if err := r.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, obj); err != nil { - t.Fatalf("get %T for resourceVersion: %v", obj, err) - } - return obj.GetResourceVersion() -} - -func ptrBool(v bool) *bool { return &v } - -type getObservingClient struct { - client.Client - onGet func(types.NamespacedName, client.Object) -} - -func (c *getObservingClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - err := c.Client.Get(ctx, key, obj, opts...) - if c.onGet != nil { - // Record attempts, including NotFound, so the watch drain sees stale - // queued reconciles that would recreate a deleted child. - c.onGet(types.NamespacedName(key), obj) - } - return err -} - -// pollDeployment waits for a Deployment to exist at key, returning its UID. -func pollDeployment(t *testing.T, k8s client.Client, key types.NamespacedName, what string) string { - t.Helper() - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var dep appsv1.Deployment - if err := k8s.Get(context.Background(), key, &dep); err == nil { - return string(dep.UID) - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("timed out waiting for deployment to %s", what) - return "" -} - -func setDeploymentStatus(t *testing.T, r *CacheBackendReconciler, name, ns string, mutate func(*appsv1.Deployment)) { - t.Helper() - dep := getDeployment(t, r, name, ns) - mutate(dep) - if err := r.Status().Update(context.Background(), dep); err != nil { - t.Fatalf("update deployment status: %v", err) - } -} - -func TestIntegrationCacheBackendReconcile(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, _ := startEnv(t) - r := &CacheBackendReconciler{Client: k8s, Scheme: scheme, Log: logr.Discard()} - ctx := context.Background() - - t.Run("LMCacheServerWorkloadShape", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Replicas = ptrInt32(2) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - dep := getDeployment(t, r, "cache", ns) - if dep.Spec.Replicas == nil || *dep.Spec.Replicas != 2 { - t.Fatalf("replicas = %v, want 2", dep.Spec.Replicas) - } - // Selector is a subset of the pod template labels. - sel := dep.Spec.Selector.MatchLabels - podLabels := dep.Spec.Template.Labels - for k, v := range sel { - if podLabels[k] != v { - t.Fatalf("selector label %s=%s missing from pod labels %v", k, v, podLabels) - } - } - - if len(dep.Spec.Template.Spec.Containers) != 1 { - t.Fatalf("containers = %d, want 1", len(dep.Spec.Template.Spec.Containers)) - } - c := dep.Spec.Template.Spec.Containers[0] - if c.Name != "lmcache-server" { - t.Fatalf("container name = %q, want lmcache-server", c.Name) - } - if !strings.Contains(c.Image, "lmcache/standalone") { - t.Fatalf("default image = %q, want the lmcache/standalone reference image", c.Image) - } - if !containsStr(c.Command, "lmcache_server") { - t.Fatalf("command = %v, want lmcache_server", c.Command) - } - if !containsStr(c.Args, "65432") || !containsStr(c.Args, "cpu") || !containsStr(c.Args, "0.0.0.0") { - t.Fatalf("args = %v, want [host port storage]", c.Args) - } - if len(c.Ports) != 1 || c.Ports[0].ContainerPort != 65432 || c.Ports[0].Protocol != corev1.ProtocolTCP { - t.Fatalf("ports = %v, want a single TCP port on 65432", c.Ports) - } - if c.ReadinessProbe == nil || c.ReadinessProbe.TCPSocket == nil { - t.Fatalf("readiness probe = %+v, want a TCP-socket probe on the lmcache port", c.ReadinessProbe) - } - - svc := getService(t, r, "cache", ns) - if svc.Spec.Type != corev1.ServiceTypeClusterIP || svc.Spec.ClusterIP == "" { - t.Fatalf("service type/clusterIP = %q/%q, want ClusterIP with an allocated IP", svc.Spec.Type, svc.Spec.ClusterIP) - } - if len(svc.Spec.Ports) != 1 || svc.Spec.Ports[0].Port != 65432 { - t.Fatalf("service ports = %v, want a single port on 65432", svc.Spec.Ports) - } - - // Real API-server pod defaulting is applied to the stored template. - spec := dep.Spec.Template.Spec - if spec.RestartPolicy == "" || spec.DNSPolicy == "" { - t.Fatalf("expected pod defaulting, got restartPolicy=%q dnsPolicy=%q", spec.RestartPolicy, spec.DNSPolicy) - } - }) - - t.Run("MooncakeMasterWorkloadShape", func(t *testing.T) { - // Mooncake provider contract against a real apiserver: the canonical - // remote binding reconciles into a mooncake_master Deployment + Service, - // and status.endpoint resolves the master's RPC port. - ns := freshNS(t, k8s) - cb := mooncakeBackend("cache", ns) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - reconcile(t, r, "cache", ns) - - dep := getDeployment(t, r, "cache", ns) - if len(dep.Spec.Template.Spec.Containers) != 1 { - t.Fatalf("containers = %d, want 1", len(dep.Spec.Template.Spec.Containers)) - } - c := dep.Spec.Template.Spec.Containers[0] - if c.Name != "mooncake-master" { - t.Fatalf("container name = %q, want mooncake-master", c.Name) - } - if !strings.Contains(c.Image, "mooncake") { - t.Fatalf("default image = %q, want the Mooncake reference image", c.Image) - } - if !containsStr(c.Command, "mooncake_master") { - t.Fatalf("command = %v, want mooncake_master", c.Command) - } - if !containsStr(c.Args, "--rpc_port=50051") { - t.Fatalf("args = %v, want --rpc_port=50051", c.Args) - } - if len(c.Ports) == 0 || c.Ports[0].ContainerPort != 50051 || c.Ports[0].Protocol != corev1.ProtocolTCP { - t.Fatalf("first port = %v, want the RPC port 50051 first", c.Ports) - } - if c.ReadinessProbe == nil || c.ReadinessProbe.TCPSocket == nil { - t.Fatalf("readiness probe = %+v, want a TCP-socket probe on the RPC port", c.ReadinessProbe) - } - - svc := getService(t, r, "cache", ns) - if len(svc.Spec.Ports) == 0 || svc.Spec.Ports[0].Port != 50051 { - t.Fatalf("service first port = %v, want RPC port 50051 first", svc.Spec.Ports) - } - - got := getBackend(t, r, "cache", ns) - wantEndpoint := "cache." + ns + ".svc.cluster.local:50051" - if got.Status.Endpoint != wantEndpoint { - t.Fatalf("status.endpoint = %q, want %q", got.Status.Endpoint, wantEndpoint) - } - }) - - t.Run("StatusEndpointAndObservedGeneration", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - cb := getBackend(t, r, "cache", ns) - wantEndpoint := fmt.Sprintf("cache.%s.svc.cluster.local:65432", ns) - if cb.Status.Endpoint != wantEndpoint { - t.Fatalf("status.endpoint = %q, want %q", cb.Status.Endpoint, wantEndpoint) - } - if cb.Status.ObservedGeneration != cb.Generation { - t.Fatalf("observedGeneration = %d, want %d", cb.Status.ObservedGeneration, cb.Generation) - } - if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond == nil { - t.Fatalf("Ready condition missing") - } - }) - - t.Run("OwnerReferencesDriveGC", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - for _, obj := range []client.Object{getDeployment(t, r, "cache", ns), getService(t, r, "cache", ns)} { - owner := metav1.GetControllerOf(obj) - if owner == nil || owner.Kind != "CacheBackend" || owner.Name != "cache" { - t.Fatalf("%T controller owner = %+v", obj, owner) - } - if owner.Controller == nil || !*owner.Controller { - t.Fatalf("%T owner Controller flag not set", obj) - } - if owner.BlockOwnerDeletion == nil || !*owner.BlockOwnerDeletion { - t.Fatalf("%T owner BlockOwnerDeletion not set (needed for GC)", obj) - } - } - }) - - t.Run("NoChurnAgainstRealDefaulting", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - // Two reconciles to converge any first-write differences before the RV snapshot. - reconcile(t, r, "cache", ns) - depRV := getRV(t, r, "cache", ns, &appsv1.Deployment{}) - svcRV := getRV(t, r, "cache", ns, &corev1.Service{}) - reconcile(t, r, "cache", ns) - if got := getRV(t, r, "cache", ns, &appsv1.Deployment{}); got != depRV { - t.Fatalf("deployment churned: RV %s -> %s", depRV, got) - } - if got := getRV(t, r, "cache", ns, &corev1.Service{}); got != svcRV { - t.Fatalf("service churned: RV %s -> %s", svcRV, got) - } - }) - - t.Run("ReadyConditionTransitions", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Replicas = ptrInt32(2) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - if cond := findCondition(getBackend(t, r, "cache", ns).Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonRolloutInProgress { - t.Fatalf("fresh Ready condition = %+v, want False/RolloutInProgress", cond) - } - - // Mid-rollout: generation observed but updated replicas lag -> Ready=False/RolloutInProgress. - setDeploymentStatus(t, r, "cache", ns, func(d *appsv1.Deployment) { - d.Status.ObservedGeneration = d.Generation - d.Status.Replicas = 2 - d.Status.UpdatedReplicas = 1 - d.Status.AvailableReplicas = 2 - d.Status.ReadyReplicas = 2 - }) - reconcile(t, r, "cache", ns) - if cond := findCondition(getBackend(t, r, "cache", ns).Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonRolloutInProgress { - t.Fatalf("mid-rollout Ready condition = %+v, want False/RolloutInProgress", cond) - } - - // Fully rolled out -> Ready=True. - setDeploymentStatus(t, r, "cache", ns, func(d *appsv1.Deployment) { - d.Status.ObservedGeneration = d.Generation - d.Status.Replicas = 2 - d.Status.UpdatedReplicas = 2 - d.Status.AvailableReplicas = 2 - d.Status.ReadyReplicas = 2 - }) - reconcile(t, r, "cache", ns) - cb = getBackend(t, r, "cache", ns) - if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionTrue { - t.Fatalf("Ready condition = %+v, want True", cond) - } - - // Rolled out but replicas unavailable -> Ready=False/ReplicasUnavailable. - setDeploymentStatus(t, r, "cache", ns, func(d *appsv1.Deployment) { - d.Status.ObservedGeneration = d.Generation - d.Status.Replicas = 2 - d.Status.UpdatedReplicas = 2 - d.Status.AvailableReplicas = 1 - d.Status.ReadyReplicas = 1 - }) - reconcile(t, r, "cache", ns) - if cond := findCondition(getBackend(t, r, "cache", ns).Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionFalse || cond.Reason != conditionReasonReplicasUnavailable { - t.Fatalf("unavailable Ready condition = %+v, want False/ReplicasUnavailable", cond) - } - }) - - t.Run("ZeroReplicasNotReady", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Replicas = ptrInt32(0) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - setDeploymentStatus(t, r, "cache", ns, func(d *appsv1.Deployment) { - d.Status.ObservedGeneration = d.Generation - }) - reconcile(t, r, "cache", ns) - cb = getBackend(t, r, "cache", ns) - if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond == nil || cond.Status != metav1.ConditionFalse { - t.Fatalf("Ready condition = %+v, want False for zero replicas", cond) - } - }) - - t.Run("ServerImageOverrideAndUpdate", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v1" - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - if got := getDeployment(t, r, "cache", ns).Spec.Template.Spec.Containers[0].Image; got != "example.com/lmcache-server:v1" { - t.Fatalf("image = %q, want override", got) - } - - live := getBackend(t, r, "cache", ns) - live.Spec.RemoteStorage.LMCacheServer.Image = "example.com/lmcache-server:v2" - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("update image: %v", err) - } - reconcile(t, r, "cache", ns) - if got := getDeployment(t, r, "cache", ns).Spec.Template.Spec.Containers[0].Image; got != "example.com/lmcache-server:v2" { - t.Fatalf("updated image = %q, want v2", got) - } - }) - - t.Run("ReplicaScale", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Replicas = ptrInt32(1) - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - live := getBackend(t, r, "cache", ns) - live.Spec.Replicas = ptrInt32(4) - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("update replicas: %v", err) - } - reconcile(t, r, "cache", ns) - if got := getDeployment(t, r, "cache", ns).Spec.Replicas; got == nil || *got != 4 { - t.Fatalf("replicas = %v, want 4", got) - } - }) - - t.Run("PodOverrideUpdate", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - live := getBackend(t, r, "cache", ns) - live.Spec.Template = &cachev1alpha1.CacheBackendPodSpecOverride{ - NodeSelector: map[string]string{"accelerator": "h100"}, - ServiceAccountName: "backend-sa", - } - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("update overrides: %v", err) - } - reconcile(t, r, "cache", ns) - spec := getDeployment(t, r, "cache", ns).Spec.Template.Spec - if spec.NodeSelector["accelerator"] != "h100" || spec.ServiceAccountName != "backend-sa" { - t.Fatalf("overrides not reconciled: nodeSelector=%v sa=%q", spec.NodeSelector, spec.ServiceAccountName) - } - }) - - t.Run("ServicePortDriftCorrected", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - svc := getService(t, r, "cache", ns) - // Drift the owned Service out-of-band: change the port number. - svc.Spec.Ports[0].Port = 9999 - if err := k8s.Update(ctx, svc); err != nil { - t.Fatalf("drift service: %v", err) - } - reconcile(t, r, "cache", ns) - svc = getService(t, r, "cache", ns) - if svc.Spec.Ports[0].Port != 65432 { - t.Fatalf("service port = %d, want 65432 restored after drift", svc.Spec.Ports[0].Port) - } - }) - - t.Run("HPACreatedAndUpdatedAndDeleted", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: ptrInt32(2), - MaxReplicas: 5, - TargetCPUUtilizationPercent: ptrInt32(60), - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - - // HPA created and points at the managed Deployment. - hpa := getHPA(t, r, "cache", ns) - if hpa.Spec.ScaleTargetRef.Kind != "Deployment" || hpa.Spec.ScaleTargetRef.Name != "cache" { - t.Fatalf("HPA target = %+v, want Deployment/cache", hpa.Spec.ScaleTargetRef) - } - if hpa.Spec.MinReplicas == nil || *hpa.Spec.MinReplicas != 2 || hpa.Spec.MaxReplicas != 5 { - t.Fatalf("HPA min/max = %v/%d, want 2/5", hpa.Spec.MinReplicas, hpa.Spec.MaxReplicas) - } - // When autoscaling is set the lmcache-server container carries CPU requests - // (the utilization denominator the HPA needs). - if cpu := getDeployment(t, r, "cache", ns).Spec.Template.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU]; cpu.IsZero() { - t.Fatalf("autoscaling backend should request CPU on the container (HPA denominator)") - } - - // Update bounds and target — reflected on the HPA. - live := getBackend(t, r, "cache", ns) - live.Spec.Autoscaling.MinReplicas = ptrInt32(3) - live.Spec.Autoscaling.MaxReplicas = 8 - live.Spec.Autoscaling.TargetCPUUtilizationPercent = ptrInt32(75) - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("update autoscaling: %v", err) - } - reconcile(t, r, "cache", ns) - hpa = getHPA(t, r, "cache", ns) - if *hpa.Spec.MinReplicas != 3 || hpa.Spec.MaxReplicas != 8 { - t.Fatalf("HPA min/max = %v/%d, want 3/8 after update", hpa.Spec.MinReplicas, hpa.Spec.MaxReplicas) - } - - // Clear autoscaling — the HPA must be garbage-collected explicitly. - live = getBackend(t, r, "cache", ns) - live.Spec.Autoscaling = nil - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("clear autoscaling: %v", err) - } - reconcile(t, r, "cache", ns) - var hpaList autoscalingv2.HorizontalPodAutoscalerList - if err := k8s.List(ctx, &hpaList, client.InNamespace(ns)); err != nil { - t.Fatalf("list HPAs: %v", err) - } - if len(hpaList.Items) != 0 { - t.Fatalf("HPAs after clearing autoscaling = %d, want 0", len(hpaList.Items)) - } - }) - - t.Run("HPANoChurnAgainstRealDefaulting", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, autoscalingBackend("cache", ns, 2, 5, ptrInt32(60))); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - // Two reconciles to converge any first-write differences before the RV snapshot. - reconcile(t, r, "cache", ns) - hpaRV := getRV(t, r, "cache", ns, &autoscalingv2.HorizontalPodAutoscaler{}) - - reconcile(t, r, "cache", ns) - if got := getRV(t, r, "cache", ns, &autoscalingv2.HorizontalPodAutoscaler{}); got != hpaRV { - t.Fatalf("HPA churned: RV %s -> %s", hpaRV, got) - } - }) - - t.Run("CRDValidationRejectsBadAutoscaling", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := lmcacheBackend("bad", ns) - // XValidation: minReplicas must not exceed maxReplicas. - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: ptrInt32(7), - MaxReplicas: 3, - } - if err := k8s.Create(ctx, cb); err == nil { - t.Fatalf("expected CRD validation to reject minReplicas>maxReplicas") - } - }) - - t.Run("SwitchToExternalCleansUpAndMirrors", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - if _, err := getOptionalDeployment(t, r, "cache", ns); err != nil { - t.Fatalf("expected managed deployment first: %v", err) - } - - live := getBackend(t, r, "cache", ns) - live.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - live.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - live.Spec.RemoteStorage = externalLMCacheStorage("external.example.svc:8080") - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("switch to external: %v", err) - } - reconcile(t, r, "cache", ns) - - if _, err := getOptionalDeployment(t, r, "cache", ns); err == nil { - t.Fatalf("deployment should be deleted after switch to External") - } - cb := getBackend(t, r, "cache", ns) - if cb.Status.Endpoint != "external.example.svc:8080" { - t.Fatalf("status.endpoint = %q, want mirrored external endpoint", cb.Status.Endpoint) - } - // After the switch to External the controller publishes - // Ready=True with reason ExternalEndpointAccepted — admission - // acceptance of spec.remoteStorage.endpoint is the only readiness - // signal we have without provisioning a Service to probe. - ready := findCondition(cb.Status.Conditions, conditionTypeReady) - if ready == nil { - t.Fatalf("Ready condition missing after switch to External; conditions = %v", cb.Status.Conditions) - } - if ready.Status != metav1.ConditionTrue || ready.Reason != "ExternalEndpointAccepted" { - t.Fatalf("Ready condition = %+v, want Status=True Reason=ExternalEndpointAccepted", ready) - } - }) - - t.Run("ExternalCreateProducesNoWorkloadAndReady", func(t *testing.T) { - // A CacheBackend with externally owned remote storage reconciled against a real - // apiserver must (a) leave the CR's namespace free of any - // controller-rendered Deployment or Service, (b) mirror - // spec.remoteStorage.endpoint into status.endpoint verbatim, and (c) publish - // Ready=True with reason ExternalEndpointAccepted so downstream - // consumers (the future readiness gate, the indexParticipation - // poller) treat the CR as usable. - ns := freshNS(t, k8s) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-fresh", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("lm://my-cache.example:8200"), - }, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "ext-fresh", ns) - - var deps appsv1.DeploymentList - if err := k8s.List(ctx, &deps, client.InNamespace(ns)); err != nil { - t.Fatalf("list deployments: %v", err) - } - if len(deps.Items) != 0 { - t.Fatalf("deployments = %d, want 0 for External backend", len(deps.Items)) - } - var svcs corev1.ServiceList - if err := k8s.List(ctx, &svcs, client.InNamespace(ns)); err != nil { - t.Fatalf("list services: %v", err) - } - if len(svcs.Items) != 0 { - t.Fatalf("services = %d, want 0 for External backend", len(svcs.Items)) - } - - got := getBackend(t, r, "ext-fresh", ns) - if got.Status.Endpoint != "lm://my-cache.example:8200" { - t.Fatalf("status.endpoint = %q, want lm://my-cache.example:8200", got.Status.Endpoint) - } - ready := findCondition(got.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "ExternalEndpointAccepted" { - t.Fatalf("Ready condition = %+v, want Status=True Reason=ExternalEndpointAccepted", ready) - } - }) - - t.Run("ExternalAdvancesObservedGeneration", func(t *testing.T) { - ns := freshNS(t, k8s) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("ext.example.svc:8080"), - }, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "ext", ns) - got := getBackend(t, r, "ext", ns) - if got.Status.Endpoint != "ext.example.svc:8080" { - t.Fatalf("status.endpoint = %q", got.Status.Endpoint) - } - if got.Status.ObservedGeneration != got.Generation { - t.Fatalf("observedGeneration = %d, want %d", got.Status.ObservedGeneration, got.Generation) - } - }) - - t.Run("UnsupportedPairNoWorkload", func(t *testing.T) { - ns := freshNS(t, k8s) - // Both values satisfy the CRD enums, but the built-in registry has no - // vLLM+SGLangHiCache adapter. This exercises the reconciler's unmanaged - // defense-in-depth path when the validating webhook is bypassed. - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "mc", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, - }, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "mc", ns) - if _, err := getOptionalDeployment(t, r, "mc", ns); err == nil { - t.Fatalf("unmanaged type should not create a deployment") - } - got := getBackend(t, r, "mc", ns) - if got.Status.ObservedGeneration != got.Generation { - t.Fatalf("observedGeneration not advanced for unmanaged type") - } - }) - - t.Run("SwitchToStatefulSetKindCleansUpAndClearsStatus", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(ctx, lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create: %v", err) - } - reconcile(t, r, "cache", ns) - if getBackend(t, r, "cache", ns).Status.Endpoint == "" { - t.Fatalf("expected published endpoint first") - } - - live := getBackend(t, r, "cache", ns) - live.Spec.DeploymentKind = cachev1alpha1.CacheBackendDeploymentKindStatefulSet - if err := k8s.Update(ctx, live); err != nil { - t.Fatalf("switch kind: %v", err) - } - reconcile(t, r, "cache", ns) - - if _, err := getOptionalDeployment(t, r, "cache", ns); err == nil { - t.Fatalf("deployment should be deleted after switch to StatefulSet kind") - } - cb := getBackend(t, r, "cache", ns) - if cb.Status.Endpoint != "" { - t.Fatalf("status.endpoint = %q, want cleared", cb.Status.Endpoint) - } - if cond := findCondition(cb.Status.Conditions, conditionTypeReady); cond != nil { - t.Fatalf("Ready condition = %+v, want removed", cond) - } - }) - - t.Run("FailOpenStatusMirrorsSpec", func(t *testing.T) { - ns := freshNS(t, k8s) - // Default (no integration spec): status.failOpen mirrors the API default (true). - if err := k8s.Create(ctx, lmcacheBackend("def", ns)); err != nil { - t.Fatalf("create default: %v", err) - } - reconcile(t, r, "def", ns) - if got := getBackend(t, r, "def", ns).Status.FailOpen; got == nil || !*got { - t.Fatalf("default status.failOpen = %v, want true", got) - } - // Explicit fail-closed: status.failOpen reflects it. - strict := lmcacheBackend("strict", ns) - strict.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: ptrBool(false)} - if err := k8s.Create(ctx, strict); err != nil { - t.Fatalf("create strict: %v", err) - } - reconcile(t, r, "strict", ns) - if got := getBackend(t, r, "strict", ns).Status.FailOpen; got == nil || *got { - t.Fatalf("strict status.failOpen = %v, want false", got) - } - }) - - t.Run("CanonicalRuntimeRouting", func(t *testing.T) { - ns := freshNS(t, k8s) - up := lmcacheBackend("up", ns) - up.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - if err := k8s.Create(ctx, up); err != nil { - t.Fatalf("create VLLM: %v", err) - } - reconcile(t, r, "up", ns) - if _, err := getOptionalDeployment(t, r, "up", ns); err != nil { - t.Fatalf("VLLM (uppercase) should match the vllm adapter and produce a Deployment: %v", err) - } - - // SGLang+LMCache is a shipping adapter. Pair it with managed Redis, - // the remote-storage protocol that the adapter accepts, and verify the - // composed registries render its managed storage Deployment. - sg := lmcacheBackend("sg", ns) - sg.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - sg.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, - } - sg.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if err := k8s.Create(ctx, sg); err != nil { - t.Fatalf("create sglang: %v", err) - } - reconcile(t, r, "sg", ns) - if _, err := getOptionalDeployment(t, r, "sg", ns); err != nil { - t.Fatalf("sglang now has a shipping adapter; expected a managed Deployment: %v", err) - } - - }) - - t.Run("MissingObjectIsNoError", func(t *testing.T) { - ns := freshNS(t, k8s) - reconcile(t, r, "does-not-exist", ns) - }) -} - -// TestIntegrationEnginePodEvents exercises the engine-pod-events controller -// against a real apiserver. The controller's contract is "emit a Normal -// InjectedByCacheBackend Event on every pod the mutating webhook stamped -// with inferencecache.io/injected-by". The user-visible promise is that -// `kubectl describe pod` surfaces the event, and describe filters events -// by involvedObject.uid — so this test asserts the recorded events carry -// the persisted Pod UID, not just the name. -// -// (This is the regression the webhook-recorded approach would have -// broken: at admission time the apiserver hasn't assigned the UID yet, -// so an event recorded from the webhook lands with involvedObject.uid="" -// and is invisible under describe.) -func TestIntegrationEnginePodEvents(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, cfg := startEnv(t) - - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - Controller: config.Controller{SkipNameValidation: ptrBool(true)}, - }) - if err != nil { - t.Fatalf("new manager: %v", err) - } - if err := (&EnginePodEventsReconciler{ - Client: mgr.GetClient(), - Log: logr.Discard(), - }).SetupWithManager(mgr); err != nil { - t.Fatalf("setup with manager: %v", err) - } - - mgrCtx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - go func() { - if err := mgr.Start(mgrCtx); err != nil { - t.Logf("manager stopped: %v", err) - } - }() - if !mgr.GetCache().WaitForCacheSync(mgrCtx) { - t.Fatalf("cache did not sync") - } - - ns := freshNS(t, k8s) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "primary", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }, - } - if err := k8s.Create(context.Background(), cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - // The apiserver assigns cb.UID on Create. The injected-by-uid - // annotation below pins that UID so the events controller's UID - // match passes — without it, the controller would skip emission - // per the failurePolicy=Ignore forgery guard. - if cb.UID == "" { - t.Fatalf("apiserver returned empty UID for persisted CacheBackend — envtest invariant broken") - } - - // Create a pod with the injected-by annotations the webhook would - // have stamped. The webhook is NOT installed in this test — we are - // exercising the controller's behavior on a pod that LOOKS like one - // the webhook produced. - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "engine-a", - Namespace: ns, - Annotations: map[string]string{ - "inferencecache.io/injected-by": ns + "/" + cb.Name, - "inferencecache.io/injected-by-uid": string(cb.UID), - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "engine", - Image: "registry.example.com/vllm:test", - }}, - }, - } - if err := k8s.Create(context.Background(), pod); err != nil { - t.Fatalf("create pod: %v", err) - } - if pod.UID == "" { - t.Fatalf("apiserver returned empty UID for persisted pod — envtest invariant broken") - } - - skipped := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "engine-skipped", - Namespace: ns, - Annotations: map[string]string{ - podwebhook.AnnotationSkip: "true", - podwebhook.AnnotationInjectSkipped: podwebhook.InjectSkippedReasonSkipAnnotation, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "engine", Image: "registry.example.com/vllm:test"}}, - }, - } - if err := k8s.Create(context.Background(), skipped); err != nil { - t.Fatalf("create skipped pod: %v", err) - } - if skipped.UID == "" { - t.Fatalf("apiserver returned empty UID for skipped pod; envtest invariant broken") - } - - // An unannotated pod that should NOT generate an event. Pins the - // predicate filtering. - unrelated := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: ns}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "x", Image: "x"}}, - }, - } - if err := k8s.Create(context.Background(), unrelated); err != nil { - t.Fatalf("create unrelated pod: %v", err) - } - - // Poll for the InjectedByCacheBackend event on the persisted pod's - // UID. The events.EventRecorder broadcasts asynchronously, so the - // event lags pod creation by a few hundred ms. - deadline := time.Now().Add(20 * time.Second) - var sawInjected bool - var sawSkipped bool - var sawSpurious bool - for time.Now().Before(deadline) && (!sawInjected || !sawSkipped) { - var list eventsv1.EventList - if err := k8s.List(context.Background(), &list, client.InNamespace(ns)); err == nil { - for _, ev := range list.Items { - if ev.Regarding.Kind != "Pod" { - continue - } - if ev.Regarding.UID == pod.UID && ev.Reason == "InjectedByCacheBackend" { - sawInjected = true - } - if ev.Regarding.UID == skipped.UID && ev.Reason == "SkippedByOperator" { - sawSkipped = true - } - if ev.Regarding.UID == unrelated.UID && ev.Reason == "InjectedByCacheBackend" { - sawSpurious = true - } - if ev.Regarding.UID == unrelated.UID && ev.Reason == "SkippedByOperator" { - sawSpurious = true - } - } - } - if sawInjected && sawSkipped { - break - } - time.Sleep(250 * time.Millisecond) - } - if !sawInjected { - t.Errorf("did not observe InjectedByCacheBackend event with involvedObject.uid=%q within timeout", pod.UID) - } - if !sawSkipped { - t.Errorf("did not observe SkippedByOperator event with involvedObject.uid=%q within timeout", skipped.UID) - } - if sawSpurious { - t.Errorf("controller emitted an engine-pod event on an unannotated pod (uid=%q); predicate failed", unrelated.UID) - } -} - -// TestIntegrationCacheBackendMatchedEnginePodsRequeueCadence verifies the -// new self-requeue cadence: a manager-driven reconciler (no Pod watch, no -// explicit reconcile() calls) must still converge `status.matchedEnginePods` -// after a pod CREATE because the previous reconcile scheduled a -// RequeueAfter when the CR's EngineSelector was non-empty. Without that -// requeue, pod birth/death would only refresh the count when an unrelated -// CR/owned-workload event fired, leaving the operator-facing column -// indefinitely stale. -func TestIntegrationCacheBackendMatchedEnginePodsRequeueCadence(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, cfg := startEnv(t) - - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - Controller: config.Controller{SkipNameValidation: ptrBool(true)}, - }) - if err != nil { - t.Fatalf("new manager: %v", err) - } - // Keep the steady cadence long and inject a short churn cadence so this - // test proves the conditional fast path: while desired Deployment replicas - // and observed matching pods disagree, the controller self-requeues quickly - // without a Pod watch. - const steadyRequeueInterval = 30 * time.Second - const churnRequeueInterval = 250 * time.Millisecond - if err := setupTestCacheBackendReconciler(mgr, &CacheBackendReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logr.Discard(), - MatchedEnginePodsRequeueInterval: steadyRequeueInterval, - MatchedEnginePodsChurnRequeueInterval: churnRequeueInterval, - }); err != nil { - t.Fatalf("setup with manager: %v", err) - } - - mgrCtx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - go func() { - if err := mgr.Start(mgrCtx); err != nil { - t.Logf("manager stopped: %v", err) - } - }() - if !mgr.GetCache().WaitForCacheSync(mgrCtx) { - t.Fatalf("cache did not sync") - } - - ns := freshNS(t, k8s) - sel := map[string]string{"app": "test-engine"} - engineDep := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: "engine", Namespace: ns}, - Spec: appsv1.DeploymentSpec{ - Replicas: ptrInt32(2), - Selector: &metav1.LabelSelector{MatchLabels: sel}, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{Labels: sel}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "engine", Image: "registry.example.com/vllm:test"}}, - }, - }, - }, - } - if err := k8s.Create(context.Background(), engineDep); err != nil { - t.Fatalf("create engine Deployment: %v", err) - } - cb := lmcacheBackend("cache", ns) - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: sel} - if err := k8s.Create(context.Background(), cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - // Helper to read the live count. - read := func() *int32 { - var live cachev1alpha1.CacheBackend - if err := k8s.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: ns}, &live); err != nil { - t.Fatalf("get CacheBackend: %v", err) - } - return live.Status.MatchedEnginePods - } - waitForCount := func(want int32, what string) { - t.Helper() - // With the injected churn cadence (250ms), the next requeue - // after a pod change lands well under a second. Add 5s of - // envtest-jitter slack on top. - deadline := time.Now().Add(churnRequeueInterval + 5*time.Second) - for time.Now().Before(deadline) { - got := read() - if got != nil && *got == want { - return - } - time.Sleep(500 * time.Millisecond) - } - t.Fatalf("status.matchedEnginePods did not converge to %d (%s) within timeout; last value = %v", want, what, read()) - } - - // First reconcile (manager-driven; no explicit reconcile() call): - // no matching pods → expect 0. - waitForCount(0, "no matching pods") - - // Create matching pods AFTER the initial reconcile and verify the - // count catches up WITHOUT us forcing a reconcile. The self- - // requeue cadence is what makes this work. - for i := 0; i < 2; i++ { - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("engine-%d", i), - Namespace: ns, - Labels: sel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{Name: "engine", Image: "registry.example.com/vllm:test"}}, - }, - } - if err := k8s.Create(context.Background(), p); err != nil { - t.Fatalf("create pod %s: %v", p.Name, err) - } - } - waitForCount(2, "after creating 2 matching pods") -} - -func TestIntegrationCacheBackendEngineSelectorUnmatchedDiagnostics(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, cfg := startEnv(t) - - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - Controller: config.Controller{SkipNameValidation: ptrBool(true)}, - }) - if err != nil { - t.Fatalf("new manager: %v", err) - } - if err := setupTestCacheBackendReconciler(mgr, &CacheBackendReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logr.Discard(), - }); err != nil { - t.Fatalf("setup with manager: %v", err) - } - - mgrCtx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - go func() { - if err := mgr.Start(mgrCtx); err != nil { - t.Logf("manager stopped: %v", err) - } - }() - if !mgr.GetCache().WaitForCacheSync(mgrCtx) { - t.Fatalf("cache did not sync") - } - - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "definitely-not-present"}, - } - if err := k8s.Create(context.Background(), cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - var lastMsg string - var sawEvent bool - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var live cachev1alpha1.CacheBackend - if err := k8s.Get(context.Background(), types.NamespacedName{Name: "cache", Namespace: ns}, &live); err != nil { - t.Fatalf("get CacheBackend: %v", err) - } - lastMsg = live.Status.EngineSelectorMessage - - var list eventsv1.EventList - if err := k8s.List(context.Background(), &list, client.InNamespace(ns)); err == nil { - for _, ev := range list.Items { - if ev.Regarding.Name == "cache" && - ev.Regarding.Kind == "CacheBackend" && - ev.Reason == eventReasonEngineSelectorUnmatched { - sawEvent = true - } - } - } - - if live.Status.MatchedEnginePods != nil && - *live.Status.MatchedEnginePods == 0 && - strings.Contains(lastMsg, "spec.engineSelector.matchLabels={app:definitely-not-present}") && - strings.Contains(lastMsg, "no Pods in namespace match") && - sawEvent { - return - } - time.Sleep(250 * time.Millisecond) - } - - t.Fatalf("unmatched selector diagnostics did not converge: message=%q sawEvent=%v", lastMsg, sawEvent) -} - -// TestIntegrationCacheBackendMatchedEnginePods exercises the -// status.matchedEnginePods writer against a real apiserver: the count -// reflects the live pod inventory in the CR's namespace, ignores pods in -// other namespaces, and survives pod birth/death between reconciles. -// -// The writer counts at reconcile cadence (no Pod watch); the test therefore -// drives reconcile() explicitly after each pod mutation. -func TestIntegrationCacheBackendMatchedEnginePods(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, _ := startEnv(t) - r := &CacheBackendReconciler{Client: k8s, Scheme: scheme, Log: logr.Discard()} - ctx := context.Background() - - createPod := func(t *testing.T, namespace, name string, podLabels map[string]string) { - t.Helper() - p := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: podLabels}, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "engine", - Image: "registry.example.com/vllm:test", - }}, - }, - } - if err := k8s.Create(ctx, p); err != nil { - t.Fatalf("create pod %s/%s: %v", namespace, name, err) - } - } - - ns := freshNS(t, k8s) - other := freshNS(t, k8s) - sel := map[string]string{"app": "test-engine"} - - cb := lmcacheBackend("cache", ns) - cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: sel} - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - // No matching pods yet → after first reconcile the field is an - // observed 0 (not nil, which would mean "not yet computed"). - reconcile(t, r, "cache", ns) - if got := getBackend(t, r, "cache", ns).Status.MatchedEnginePods; got == nil || *got != 0 { - t.Fatalf("with zero matching pods: matchedEnginePods = %v, want 0", got) - } - - // Three matching pods land. A pod with the same labels in a - // different namespace and a non-matching pod in this one must not - // inflate the count. - createPod(t, ns, "engine-1", sel) - createPod(t, ns, "engine-2", sel) - createPod(t, ns, "engine-3", sel) - createPod(t, ns, "router-1", map[string]string{"app": "router"}) - createPod(t, other, "engine-foreign", sel) - - reconcile(t, r, "cache", ns) - if got := getBackend(t, r, "cache", ns).Status.MatchedEnginePods; got == nil || *got != 3 { - t.Fatalf("after creating 3 matching pods: matchedEnginePods = %v, want 3", got) - } - - // Delete one of the matching pods (force, since envtest has no - // kubelet and pods never go past Pending → no graceful delete). - zero := int64(0) - if err := k8s.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-1", Namespace: ns}}, - &client.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { - t.Fatalf("delete pod: %v", err) - } - // Tight poll: envtest's delete is async; reconcile until the count - // catches up (or the deadline fires). - deadline := time.Now().Add(10 * time.Second) - var last *int32 - for time.Now().Before(deadline) { - reconcile(t, r, "cache", ns) - last = getBackend(t, r, "cache", ns).Status.MatchedEnginePods - if last != nil && *last == 2 { - return - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("after deleting 1 of 3 matching pods: matchedEnginePods = %v, want 2", last) -} - -// TestIntegrationCacheBackendWatch runs a real manager so the Owns(...) watches -// are exercised end to end: deleting managed Deployment/HPA children re-triggers -// reconcile and the controller recreates them. -func TestIntegrationCacheBackendWatch(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, cfg := startEnv(t) - - // SkipNameValidation: multiple manager-based subtests in the same test binary - // would otherwise collide on the global controller-name registry. - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - Controller: config.Controller{SkipNameValidation: ptrBool(true)}, - }) - if err != nil { - t.Fatalf("new manager: %v", err) - } - hpaGets := make(chan types.NamespacedName, 100) - observedClient := &getObservingClient{ - Client: mgr.GetClient(), - onGet: func(key types.NamespacedName, obj client.Object) { - if _, ok := obj.(*autoscalingv2.HorizontalPodAutoscaler); !ok { - return - } - select { - case hpaGets <- key: - default: - } - }, - } - if err := setupTestCacheBackendReconciler(mgr, &CacheBackendReconciler{ - Client: observedClient, - Scheme: mgr.GetScheme(), - Log: logr.Discard(), - }); err != nil { - t.Fatalf("setup with manager: %v", err) - } - - mgrCtx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - go func() { - if err := mgr.Start(mgrCtx); err != nil { - t.Logf("manager stopped: %v", err) - } - }() - if !mgr.GetCache().WaitForCacheSync(mgrCtx) { - t.Fatalf("cache did not sync") - } - - waitForHPA := func(t *testing.T, key types.NamespacedName, what string) string { - t.Helper() - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var hpa autoscalingv2.HorizontalPodAutoscaler - if err := k8s.Get(context.Background(), key, &hpa); err == nil { - return string(hpa.UID) - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("timed out waiting for HPA to %s", what) - return "" - } - waitForObservedGeneration := func(t *testing.T, key types.NamespacedName, want int64) { - t.Helper() - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var live cachev1alpha1.CacheBackend - if err := k8s.Get(context.Background(), key, &live); err == nil && live.Status.ObservedGeneration >= want { - return - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("timed out waiting for CacheBackend %s/%s observedGeneration to reach %d", key.Namespace, key.Name, want) - } - drainHPAGetEvents := func() { - for { - select { - case <-hpaGets: - default: - return - } - } - } - waitForHPAGet := func(t *testing.T, key types.NamespacedName, what string) { - t.Helper() - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - select { - case got := <-hpaGets: - if got == key { - return - } - case <-time.After(200 * time.Millisecond): - } - } - t.Fatalf("timed out waiting for reconciler to get HPA %s/%s (%s)", key.Namespace, key.Name, what) - } - waitForQuietHPAGets := func(t *testing.T, key types.NamespacedName, quietFor time.Duration) { - t.Helper() - deadline := time.Now().Add(20 * time.Second) - quietUntil := time.Now().Add(quietFor) - for { - if time.Now().After(deadline) { - t.Fatalf("timed out waiting for reconciler HPA gets on %s/%s to go quiet", key.Namespace, key.Name) - } - remainingQuiet := time.Until(quietUntil) - if remainingQuiet <= 0 { - return - } - select { - case got := <-hpaGets: - if got == key { - quietUntil = time.Now().Add(quietFor) - } - case <-time.After(remainingQuiet): - return - } - } - } - - t.Run("OwnsDeploymentWatchRecreatesDeletedChild", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(context.Background(), lmcacheBackend("cache", ns)); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - key := types.NamespacedName{Name: "cache", Namespace: ns} - originalUID := pollDeployment(t, k8s, key, "be created by the manager") - - // Delete the child; the Owns() watch must re-trigger reconcile and recreate it. - if err := k8s.Delete(context.Background(), &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: ns}, - }); err != nil { - t.Fatalf("delete deployment: %v", err) - } - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var dep appsv1.Deployment - if err := k8s.Get(context.Background(), key, &dep); err == nil && string(dep.UID) != originalUID { - return // recreated with a new UID — Owns(Deployment) watch re-trigger confirmed - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("deployment was not recreated after deletion (Owns watch did not re-trigger)") - }) - - t.Run("OwnsHPAWatchRecreatesDeletedChild", func(t *testing.T) { - ns := freshNS(t, k8s) - if err := k8s.Create(context.Background(), autoscalingBackend("cache", ns, 2, 5, ptrInt32(60))); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - key := types.NamespacedName{Name: "cache", Namespace: ns} - waitForHPA(t, key, "be created by the manager") - - // Drain the initial create/status/owned-child event burst before deleting - // the HPA. Otherwise an already-queued parent/Deployment reconcile could - // recreate the HPA and make this test pass even if Owns(HPA) were absent. - // - // The drain waits for a known spec-driven reconcile to reach the HPA, then - // waits for HPA reads to go quiet. It deliberately does not require a - // status-only parent reconcile; that keeps the test valid if the parent - // watch is ever narrowed to generation-changing updates. - drainHPAGetEvents() - var live cachev1alpha1.CacheBackend - if err := k8s.Get(context.Background(), key, &live); err != nil { - t.Fatalf("get CacheBackend before drain update: %v", err) - } - beforeGeneration := live.Generation - if live.Spec.Observation == nil { - live.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{} - } - live.Spec.Observation.ModelID = "test-drain-" + time.Now().Format(time.RFC3339Nano) - if err := k8s.Update(context.Background(), &live); err != nil { - t.Fatalf("update CacheBackend to drain initial queue: %v", err) - } - if live.Generation <= beforeGeneration { - t.Fatalf("drain update did not advance generation: %d -> %d", beforeGeneration, live.Generation) - } - waitForObservedGeneration(t, key, live.Generation) - waitForHPAGet(t, key, "drain spec update reconcile") - waitForQuietHPAGets(t, key, 500*time.Millisecond) - - originalUID := waitForHPA(t, key, "remain after the drain reconcile") - - // Delete the child; the Owns() watch must re-trigger reconcile and recreate it. - if err := k8s.Delete(context.Background(), &autoscalingv2.HorizontalPodAutoscaler{ - ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: ns}, - }); err != nil { - t.Fatalf("delete HPA: %v", err) - } - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var hpa autoscalingv2.HorizontalPodAutoscaler - if err := k8s.Get(context.Background(), key, &hpa); err == nil && string(hpa.UID) != originalUID { - return // recreated with a new UID — Owns(HPA) watch re-trigger confirmed - } - time.Sleep(200 * time.Millisecond) - } - t.Fatalf("HPA was not recreated after deletion (Owns watch did not re-trigger)") - }) -} - -// TestIntegrationCacheIndexPollerProjectsParticipation exercises the poller -// against a real apiserver to confirm that Status().Patch on CacheBackend -// applies the indexParticipation projection (pod-label-based attribution), -// and that a steady snapshot does not churn the backend's resourceVersion -// (the no-churn invariant under real apiserver defaulting). Catches the -// class of bug a fake client misses — the fake client skips apiserver -// defaulting that can flip semantic equality on round-trip and cause -// spurious writes. -func TestIntegrationCacheIndexPollerProjectsParticipation(t *testing.T) { - skipWithoutEnvtest(t) - k8s, _, _ := startEnv(t) - ctx := context.Background() - ns := freshNS(t, k8s) - - // Seed two CacheBackends with EngineSelectors plus an engine pod each. - // External ownership avoids managed child provisioning in this fixture — - // we are testing the poller's Status().Patch in isolation. - mkBackend := func(name string, selector map[string]string) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("external.example:6379"), - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: selector}, - }, - } - } - mkPod := func(name string, labels map[string]string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, - Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "vllm", Image: "vllm/vllm-openai:latest"}}}, - } - } - for _, obj := range []client.Object{ - mkBackend("backend-a", map[string]string{"app": "vllm-a"}), - mkBackend("backend-b", map[string]string{"app": "vllm-b"}), - mkPod("vllm-a-0", map[string]string{"app": "vllm-a"}), - mkPod("vllm-b-0", map[string]string{"app": "vllm-b"}), - } { - if err := k8s.Create(ctx, obj); err != nil { - t.Fatalf("create %T %s: %v", obj, obj.GetName(), err) - } - } - - tEvent := time.Now().Add(-30 * time.Second).UTC().Truncate(time.Second) - var mu sync.Mutex - served := controlplaneapi.Snapshot{ - Replicas: []controlplaneapi.ReplicaSnapshot{ - {ReplicaID: "vllm-a-0", Tenant: ns, PrefixCount: 4, LastEventAt: tEvent}, - {ReplicaID: "vllm-b-0", Tenant: ns, PrefixCount: 1, LastEventAt: tEvent}, - }, - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - mu.Lock() - defer mu.Unlock() - _ = json.NewEncoder(w).Encode(served) - })) - defer srv.Close() - - p := &CacheIndexPoller{Client: k8s, SnapshotURL: srv.URL, HTTPClient: srv.Client(), Name: "cluster-default"} - if err := p.refresh(ctx); err != nil { - t.Fatalf("first refresh: %v", err) - } - - a := getBackendDirect(t, k8s, "backend-a", ns) - if a.Status.IndexParticipation == nil || a.Status.IndexParticipation.PrefixCount != 4 { - t.Fatalf("backend-a participation = %+v, want prefixCount 4", a.Status.IndexParticipation) - } - rvA := a.ResourceVersion - - // Second refresh on identical snapshot → no churn (apiserver-side). - if err := p.refresh(ctx); err != nil { - t.Fatalf("second refresh: %v", err) - } - a2 := getBackendDirect(t, k8s, "backend-a", ns) - if a2.ResourceVersion != rvA { - t.Fatalf("steady snapshot churned resourceVersion (%s → %s)", rvA, a2.ResourceVersion) - } -} - -// TestIntegrationCacheIndexAcceptsUntenantedTenantRow proves the empty-string -// tenant sentinel survives a real apiserver write. Untenanted prefixes bucket -// under tenantID "" so the Σ tenants[].indexEntries == totalPrefixes invariant -// holds; that "" row flows into CacheIndex.status.tenants[], whose listMapKey is -// `id`. An empty listMapKey value is unusual, so verify the apiserver accepts it -// (a fake client would not catch a structural-schema rejection). -func TestIntegrationCacheIndexAcceptsUntenantedTenantRow(t *testing.T) { - skipWithoutEnvtest(t) - k8s, _, _ := startEnv(t) - ctx := context.Background() - - served := controlplaneapi.Snapshot{ - TotalPrefixes: 5, - Tenants: []controlplaneapi.TenantSnapshot{ - {TenantID: "", IndexEntries: 2}, // untenanted bucket - {TenantID: "team", IndexEntries: 3}, - }, - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(served) - })) - defer srv.Close() - - p := &CacheIndexPoller{Client: k8s, SnapshotURL: srv.URL, HTTPClient: srv.Client(), Name: "cluster-default"} - if err := p.refresh(ctx); err != nil { - t.Fatalf("refresh writing a CacheIndex with an empty-id tenant row: %v", err) - } - - var ci cachev1alpha1.CacheIndex - if err := k8s.Get(ctx, types.NamespacedName{Name: "cluster-default"}, &ci); err != nil { - t.Fatalf("get CacheIndex: %v", err) - } - var sawUntenanted, sawTeam bool - var sum int64 - for _, tn := range ci.Status.Tenants { - if tn.IndexEntries != nil { - sum += *tn.IndexEntries - } - switch tn.ID { - case "": - sawUntenanted = true - case "team": - sawTeam = true - } - } - if !sawUntenanted || !sawTeam { - t.Fatalf("CacheIndex tenants = %+v, want both \"\" (untenanted) and \"team\"", ci.Status.Tenants) - } - // The invariant is verifiable from the CacheIndex CR: Σ tenants[].indexEntries - // == prefixes.summary.total (2 + 3 == 5). - if sum != int64(ci.Status.Prefixes.Summary.Total) { - t.Fatalf("Σ tenants[].indexEntries = %d, want == prefixes.summary.total %d", sum, ci.Status.Prefixes.Summary.Total) - } -} - -// TestIntegrationCacheBackendPrinterColumnsRenderParticipation verifies the -// operator-facing promise: `kubectl get cachebackend` shows Prefixes and -// LastEvent columns sourced from status.indexParticipation. We hit the -// apiserver's Table content type — exactly the negotiation kubectl does -// under the hood — and assert column headers, types, and per-row cell -// values match. Catches accidental removal of the +kubebuilder:printcolumn -// markers, JSONPath drift, and renames of the underlying status fields. -func TestIntegrationCacheBackendPrinterColumnsRenderParticipation(t *testing.T) { - skipWithoutEnvtest(t) - k8s, _, cfg := startEnv(t) - ctx := context.Background() - ns := freshNS(t, k8s) - - // Two backends: one with positive participation, one drained-but-quiet. - // External ownership avoids managed child provisioning in this fixture so - // we are testing the printer-column projection from status, not the - // reconciler. - active := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "backend-a", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("lm://cache-svc:6379"), - }, - } - if err := k8s.Create(ctx, active); err != nil { - t.Fatalf("create active: %v", err) - } - quiet := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "backend-b", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage("lm://cache-svc:6379"), - }, - } - if err := k8s.Create(ctx, quiet); err != nil { - t.Fatalf("create quiet: %v", err) - } - - // Set the participation status directly via the status subresource — - // same path the poller uses, no poller in this test. - lastEvent := metav1.NewTime(time.Now().Add(-5 * time.Minute).UTC().Truncate(time.Second)) - active.Status.IndexParticipation = &cachev1alpha1.CacheBackendIndexParticipation{ - PrefixCount: 42, - LastEventAt: &lastEvent, - } - if err := k8s.Status().Update(ctx, active); err != nil { - t.Fatalf("update active status: %v", err) - } - quiet.Status.IndexParticipation = &cachev1alpha1.CacheBackendIndexParticipation{ - PrefixCount: 0, - // LastEventAt deliberately nil — kubectl renders this as . - } - if err := k8s.Status().Update(ctx, quiet); err != nil { - t.Fatalf("update quiet status: %v", err) - } - - // Hit the apiserver with the Table accept header. This is exactly what - // `kubectl get` does: the server-side rendering is what defines the - // columns the operator sees, so this is the most honest test of the - // before/after promise. - // Use the typed REST client so namespace/resource path construction and - // auth wiring exactly match what kubectl does internally. - restCfg := rest.CopyConfig(cfg) - gv := cachev1alpha1.GroupVersion - restCfg.GroupVersion = &gv - restCfg.APIPath = "/apis" - restCfg.NegotiatedSerializer = serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion() - restClient, err := rest.RESTClientFor(restCfg) - if err != nil { - t.Fatalf("build REST client: %v", err) - } - raw, err := restClient.Get(). - Namespace(ns). - Resource("cachebackends"). - SetHeader("Accept", "application/json;as=Table;v=v1;g=meta.k8s.io"). - DoRaw(ctx) - if err != nil { - t.Fatalf("apiserver Table request: %v", err) - } - - var table metav1.Table - if err := json.Unmarshal(raw, &table); err != nil { - t.Fatalf("decode Table: %v\nraw=%s", err, raw) - } - if len(table.ColumnDefinitions) == 0 || len(table.Rows) == 0 { - t.Fatalf("apiserver returned an empty Table: %+v", table) - } - - // Find the Prefixes and LastEvent columns and assert their types match - // the +kubebuilder:printcolumn markers in api/v1alpha1/cachebackend_types.go. - wantCols := map[string]string{"Prefixes": "integer", "LastEvent": "date"} - colIdx := map[string]int{} - for i, col := range table.ColumnDefinitions { - if wantType, ok := wantCols[col.Name]; ok { - if col.Type != wantType { - t.Errorf("column %q type = %q, want %q", col.Name, col.Type, wantType) - } - colIdx[col.Name] = i - } - } - for name := range wantCols { - if _, ok := colIdx[name]; !ok { - t.Fatalf("column %q missing from `kubectl get cachebackend` output", name) - } - } - - // Per-row cell assertions: active shows 42, quiet shows 0; LastEvent - // on quiet is the empty/ cell. - wantPrefixes := map[string]float64{"backend-a": 42, "backend-b": 0} - for _, row := range table.Rows { - var obj metav1.PartialObjectMetadata - if err := json.Unmarshal(row.Object.Raw, &obj); err != nil { - t.Fatalf("decode row object: %v", err) - } - expected, ok := wantPrefixes[obj.Name] - if !ok { - continue - } - gotPrefixes, ok := row.Cells[colIdx["Prefixes"]].(float64) - if !ok { - t.Fatalf("%s Prefixes cell type = %T (%v), want number", obj.Name, row.Cells[colIdx["Prefixes"]], row.Cells[colIdx["Prefixes"]]) - } - if gotPrefixes != expected { - t.Errorf("%s Prefixes cell = %v, want %v", obj.Name, gotPrefixes, expected) - } - switch obj.Name { - case "backend-a": - // Set lastEventAt → cell should be a non-empty string (the apiserver - // renders date columns as relative ages like "5m"). - cell, _ := row.Cells[colIdx["LastEvent"]].(string) - if cell == "" || cell == "" { - t.Errorf("backend-a LastEvent cell = %q, want a rendered duration", cell) - } - case "backend-b": - // Nil lastEventAt → cell is empty / ; the apiserver returns - // the empty string for a missing date field. - cell := row.Cells[colIdx["LastEvent"]] - if s, ok := cell.(string); ok && s != "" && s != "" { - t.Errorf("backend-b LastEvent cell = %q, want empty/", s) - } - } - } -} - -// TestIntegrationCacheBackendEvents runs a real manager (so the Recorder is -// auto-wired) and asserts the two transition Events the controller emits on -// status changes — FailClosedEnabled (spec.integration.failOpen flipped to -// false) and BackendDegraded (rolled out, but no available replicas) — actually -// reach the apiserver, end to end. -func TestIntegrationCacheBackendEvents(t *testing.T) { - skipWithoutEnvtest(t) - k8s, scheme, cfg := startEnv(t) - - // SkipNameValidation: multiple manager-based subtests in the same test binary - // would otherwise collide on the global controller-name registry. - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{BindAddress: "0"}, - Controller: config.Controller{SkipNameValidation: ptrBool(true)}, - }) - if err != nil { - t.Fatalf("new manager: %v", err) - } - if err := setupTestCacheBackendReconciler(mgr, &CacheBackendReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Log: logr.Discard(), - }); err != nil { - t.Fatalf("setup with manager: %v", err) - } - - mgrCtx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - go func() { - if err := mgr.Start(mgrCtx); err != nil { - t.Logf("manager stopped: %v", err) - } - }() - if !mgr.GetCache().WaitForCacheSync(mgrCtx) { - t.Fatalf("cache did not sync") - } - - // A fresh CR with spec.integration.failOpen=false: the first reconcile - // observes a true→false transition (status.failOpen defaults to true when - // unset) and emits FailClosedEnabled. - ns := freshNS(t, k8s) - cb := lmcacheBackend("cache", ns) - cb.Spec.Replicas = ptrInt32(1) - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{FailOpen: ptrBool(false)} - if err := k8s.Create(context.Background(), cb); err != nil { - t.Fatalf("create CacheBackend: %v", err) - } - - key := types.NamespacedName{Name: "cache", Namespace: ns} - pollDeployment(t, k8s, key, "be created by the manager") - - // Drive a Pending→Degraded transition by patching the Deployment status to - // rolled-out but with no available replicas. - var dep appsv1.Deployment - if err := k8s.Get(context.Background(), key, &dep); err != nil { - t.Fatalf("get dep: %v", err) - } - dep.Status.ObservedGeneration = dep.Generation - dep.Status.Replicas = 1 - dep.Status.UpdatedReplicas = 1 - dep.Status.AvailableReplicas = 0 - dep.Status.ReadyReplicas = 0 - if err := k8s.Status().Update(context.Background(), &dep); err != nil { - t.Fatalf("patch dep status: %v", err) - } - - wantReasons := map[string]bool{ - eventReasonFailClosedEnabled: false, - eventReasonBackendDegraded: false, - } - deadline := time.Now().Add(20 * time.Second) - for time.Now().Before(deadline) { - var list eventsv1.EventList - if err := k8s.List(context.Background(), &list, client.InNamespace(ns)); err == nil { - for _, ev := range list.Items { - if ev.Regarding.Name != "cache" || ev.Regarding.Kind != "CacheBackend" { - continue - } - if _, ok := wantReasons[ev.Reason]; ok { - wantReasons[ev.Reason] = true - } - } - } - allSeen := true - for _, seen := range wantReasons { - if !seen { - allSeen = false - break - } - } - if allSeen { - return - } - time.Sleep(250 * time.Millisecond) - } - for reason, seen := range wantReasons { - if !seen { - t.Errorf("did not observe Event reason=%s on CacheBackend cache/%s within timeout", reason, ns) - } - } -} diff --git a/internal/enginebinding/runtime.go b/internal/enginebinding/runtime.go index e8899a76..e0edda92 100644 --- a/internal/enginebinding/runtime.go +++ b/internal/enginebinding/runtime.go @@ -46,9 +46,3 @@ func IsValidKernelCheckMode(s string) bool { return false } } - -// EngineHostNetworkRequested reports whether the operator opted an engine pod -// using a Mooncake remote binding into host networking. -func EngineHostNetworkRequested(cache *cachev1alpha1.CacheBackend) bool { - return cache != nil && cache.Spec.Integration != nil && cache.Spec.Integration.EngineHostNetwork -} diff --git a/internal/webhook/pod/doc.go b/internal/webhook/pod/doc.go index 48a023eb..5f6829bb 100644 --- a/internal/webhook/pod/doc.go +++ b/internal/webhook/pod/doc.go @@ -11,7 +11,7 @@ // 2. picks the first whose Spec.EngineSelector.MatchLabels match the pod; // 3. resolves a runtime adapter from the controller's runtime.Registry; // 4. resolves the cache endpoint from Spec.RemoteStorage.Endpoint for -// externally owned storage or Status.Endpoint for managed providers. +// externally owned storage or Status.RemoteStorage.Endpoint for managed providers. // Endpoint-free adapters such // as native SGLang HiCache bypass this gate; and // 5. calls adapter.InjectEngineConfig(pod.Spec, binding, cache) to merge @@ -26,6 +26,6 @@ // pod produces an empty JSON-patch set and the handler does not need a // separate env-presence short-circuit. Trusting the adapter — rather than a // lenient env-only check at the handler — avoids the trap where a -// partially-wired pod (e.g. only LMCACHE_REMOTE_URL set by hand) is +// partially-wired pod (for example, only one MP connector argument set by hand) is // admitted permanently missing the rest of the wiring. package pod diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index 57f45080..ef4fb23d 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -35,10 +35,11 @@ import ( // TestWebhookOnEnvtest_EndToEnd boots a real apiserver via envtest, installs // the controller's MutatingWebhookConfiguration, starts the controller-runtime // manager with the Pod admission handler registered, then creates a -// CacheBackend whose status.endpoint is populated and a matching engine Pod. +// typed CacheBackend whose managed Redis status endpoint is populated and a +// matching engine Pod. // On admission the apiserver routes the CREATE through the webhook over the -// local serving cert and asserts the persisted pod carries the LMCache env + -// the kv-transfer-config arg the adapter writes. +// local serving cert and asserts the persisted pod carries the typed MP +// sidecar and kv-transfer-config the adapter writes. // // Skips when KUBEBUILDER_ASSETS is unset so default CI stays green. // Run locally via: @@ -141,10 +142,23 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + Port: 65432, + L1Capacity: resource.MustParse("1Gi"), + MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("2Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }}, + }, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, @@ -161,7 +175,10 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { if err := mgr.GetClient().Create(ctx, cb); err != nil { t.Fatalf("create CacheBackend: %v", err) } - cb.Status.Endpoint = "envtest-cb.default.svc.cluster.local:65432" + cb.Status.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Endpoint: "envtest-cb.default.svc.cluster.local:6379", + } if err := mgr.GetClient().Status().Update(ctx, cb); err != nil { t.Fatalf("set CacheBackend status: %v", err) } @@ -189,8 +206,14 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { t.Fatalf("get pod after create: %v", err) } - mustHaveContainerEnv(t, &got, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) - mustHaveContainerEnv(t, &got, testEnvVLLMUseV1, "1") + config := envtestArgValue(got.Spec.Containers[0].Args, "--kv-transfer-config") + if !strings.Contains(config, `"kv_connector":"LMCacheMPConnector"`) { + t.Fatalf("typed vLLM kv-transfer-config = %q", config) + } + server := envtestFindInitContainer(&got, "lmcache-mp-server") + if server == nil || !containsArgPair(server.Args, "--l2-adapter", `{"host":"envtest-cb.default.svc.cluster.local","port":6379,"type":"resp"}`) { + t.Fatalf("typed MP server does not carry managed Redis binding: %+v", server) + } if got.Annotations[AnnotationInjectedBy] != ns+"/"+cb.Name { t.Fatalf("annotation %s: got %q want %q", AnnotationInjectedBy, got.Annotations[AnnotationInjectedBy], ns+"/"+cb.Name) @@ -250,7 +273,9 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { if err := mgr.GetAPIReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: pod2.Name}, &got2); err != nil { t.Fatalf("get second pod: %v", err) } - mustHaveContainerEnv(t, &got2, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + if envtestFindInitContainer(&got2, "lmcache-mp-server") == nil { + t.Fatalf("second admitted Pod is missing typed MP sidecar: %+v", got2.Spec.InitContainers) + } skipped := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -280,9 +305,9 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { if got := gotSkipped.Annotations[AnnotationInjectedBy]; got != "" { t.Fatalf("annotation %s: got %q want absent on skipped pod", AnnotationInjectedBy, got) } - if envtestHasContainerEnv(&gotSkipped, testEnvLMCacheRemoteURL) { - t.Fatalf("skipped pod unexpectedly has %s env; webhook must not inject engine wiring when %s=true", - testEnvLMCacheRemoteURL, AnnotationSkip) + if envtestFindInitContainer(&gotSkipped, "lmcache-mp-server") != nil || + containsArgFlag(gotSkipped.Spec.Containers[0].Args, "--kv-transfer-config") { + t.Fatalf("skipped pod unexpectedly has typed MP wiring; webhook must not inject when %s=true", AnnotationSkip) } // Typed SGLang PodLocal smoke: this goes through a real apiserver so the @@ -445,36 +470,6 @@ func envtestHasContainerEnvValue(pod *corev1.Pod, name, value string) bool { return false } -// mustHaveContainerEnv fails the test if the first container's env array -// does not include name=value. -func mustHaveContainerEnv(t *testing.T, pod *corev1.Pod, name, value string) { - t.Helper() - if len(pod.Spec.Containers) == 0 { - t.Fatalf("no containers on pod %s", pod.Name) - } - for _, e := range pod.Spec.Containers[0].Env { - if e.Name == name { - if e.Value != value { - t.Fatalf("env %s on %s: got %q want %q", name, pod.Name, e.Value, value) - } - return - } - } - t.Fatalf("env %s missing on %s; have %v", name, pod.Name, pod.Spec.Containers[0].Env) -} - -func envtestHasContainerEnv(pod *corev1.Pod, name string) bool { - if len(pod.Spec.Containers) == 0 { - return false - } - for _, e := range pod.Spec.Containers[0].Env { - if e.Name == name { - return true - } - } - return false -} - // envtestFindContainer returns the container in pod with the given name, or // nil if absent. The non-envtest unit tests have a similarly named helper — // the two test files don't share state (envtest_integration_test.go skips @@ -488,6 +483,15 @@ func envtestFindContainer(pod *corev1.Pod, name string) *corev1.Container { return nil } +func envtestFindInitContainer(pod *corev1.Pod, name string) *corev1.Container { + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == name { + return &pod.Spec.InitContainers[i] + } + } + return nil +} + // envtestContainerNames returns the container names of pod for error messages. func envtestContainerNames(pod *corev1.Pod) []string { out := make([]string, len(pod.Spec.Containers)) diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index d8fabc82..35cd5daa 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -107,7 +107,7 @@ type EngineInjector struct { // Reader lists CacheBackends in the pod's namespace. Production wiring // passes the manager's APIReader (an uncached live client) — pod // CREATE is a one-shot injection opportunity, so a stale informer view - // of the owning CacheBackend (in particular a status.endpoint that + // of the owning CacheBackend (in particular remote-storage status that // lags reality) would leave the pod permanently unwired. Live reads // also avoid a cold-cache window at controller startup. Tests inject // a fake.NewClientBuilder()-derived reader, which also satisfies the @@ -188,18 +188,16 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi runtimeID, cache.Spec.Type, err)) } - // The typed LMCache topology is the Phase-1 boundary between the legacy - // in-process/flat-field wire and the final MP adapters. Never pass a typed MP - // object to a legacy adapter: doing so would silently inject the old vLLM IP - // connector or ignore the new PodLocal server settings. Until an adapter - // implements LMCacheMPRuntimeAdapter, admit the engine Pod untouched. + // A typed LMCache topology requires the runtime-specific MP compatibility + // check. A registry extension that selects LMCache without implementing this + // contract is admitted without mutation instead of receiving a partial wire. if cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" { mpAdapter, ok := adapter.(adapterruntime.LMCacheMPRuntimeAdapter) if !ok { log.V(1).Info("fail-open: selected adapter does not implement typed LMCache MP topology", "runtime", string(runtimeID), "topology", string(cache.Spec.LMCache.Topology)) return failOpen(req, &pod, fmt.Sprintf( - "runtime=%q adapter does not implement typed LMCache topology=%q (fail-open, no legacy injection)", + "runtime=%q adapter does not implement typed LMCache topology=%q (fail-open, no injection)", runtimeID, cache.Spec.LMCache.Topology)) } if err := mpAdapter.ValidateMPEnginePod(&pod, cache); err != nil { @@ -224,8 +222,8 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi } // Events-only (tier-1 routing) backends provision no server, so they publish // no endpoint — and they wire no KV connector, so they need none. The - // endpoint gate exists ONLY because the connector requires a dial target - // (an empty/malformed LMCACHE_REMOTE_URL crashes the engine at startup); an + // endpoint gate exists only because an optional remote-storage adapter + // requires a dial target; an // events-only pod injects only the observation sidecar (InjectEngineConfig // is a no-op in this mode), so bypass the gate and inject without one. // Engine-local adapters such as native SGLang HiCache have a nil binding and @@ -233,7 +231,7 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi if binding != nil && binding.Endpoint == "" && !cache.Spec.IsEventsOnly() { // The endpoint source is ownership-scoped (see effectiveEndpoint). // Three reasons we can land here: - // - managed CR: reconciler hasn't published status.endpoint + // - managed CR: reconciler hasn't published status.remoteStorage.endpoint // yet (steady-state during initial rollout). // - externally owned CR: spec.remoteStorage.endpoint is empty // (current admission rejects this; reachable only for objects that @@ -250,7 +248,7 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi // inbound pod get stripped — otherwise the events controller // would falsely emit InjectedByCacheBackend even though the // webhook bailed out without injecting. - missingField := "status.endpoint" + missingField := "status.remoteStorage.endpoint" extra := "" if storage != nil && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { missingField = "spec.remoteStorage.endpoint" @@ -266,8 +264,8 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi // No env-presence short-circuit here: the adapter is the source of truth // for the full injected contract (env + the adapter-required args/flags), // and lenient short-circuits risk admitting a pod that carries only a - // subset of the wiring (e.g. a pre-set LMCACHE_REMOTE_URL but missing the - // engine's connector flag — vLLM's --kv-transfer-config or SGLang's + // subset of the wiring (e.g. a pre-set connector argument but missing the + // engine's required MP configuration — vLLM's --kv-transfer-config or SGLang's // --enable-lmcache) permanently un-converged. Call the adapter // unconditionally; it merges idempotently (upsertEnv / upsertArgPair / // upsertFlag) and a no-op merge produces an @@ -483,7 +481,7 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi // // A CacheBackend with no EngineSelector or with an empty MatchLabels map is // skipped: a "match everything" selector at admission time would silently -// claim every pod (including the controller's own and the lmcache-server's), +// claim every pod (including control-plane and provider pods), // which is the kind of broad mutation the fail-open posture is meant to // prevent. func (h *EngineInjector) selectCacheBackend(ctx context.Context, pod *corev1.Pod) (*cachev1alpha1.CacheBackend, error) { @@ -607,15 +605,15 @@ func skipInjection(req admission.Request, pod *corev1.Pod) admission.Response { // to for the given CacheBackend. The source is ownership-scoped: // // - External ownership: spec.remoteStorage.endpoint is authoritative — the operator owns it, -// admission validates it, status.endpoint is just a reconciler +// admission validates it, status.remoteStorage.endpoint is just a reconciler // mirror that may briefly lag during an update. If a new pod // admits between an operator's spec.remoteStorage.endpoint update and the -// status patch, status would still hold the OLD value and the +// status patch, status would still hold the old value and the // pod would boot wired to the stale address; pod admission is // CREATE-only so that bad wiring is permanent. Preferring the trimmed // remoteStorage endpoint over status here avoids that race and is // consistent with admission's view of the truth. -// - Managed storage: status.endpoint is the only +// - Managed storage: status.remoteStorage.endpoint is the only // source — the reconciler builds it from the live Service it // provisions, and spec.remoteStorage.endpoint is admission-rejected for // managed ownership, so there's nothing else to fall back on. The webhook @@ -628,12 +626,9 @@ func skipInjection(req admission.Request, pod *corev1.Pod) admission.Response { // Every return path is `strings.TrimSpace`-d so a whitespace-only value // (a pre-admission CR that mirrored whitespace into status, an // externally-edited Service endpoint that picked up stray padding) is -// treated as missing and fails open instead of injecting -// `LMCACHE_REMOTE_URL=lm:// ` which the engine connector would reject -// at runtime. The reconciler already trims before publishing -// status.endpoint, but the webhook trims defensively here too so a -// race against an old controller build can't leak whitespace to the -// engine wire. +// treated as missing and fails open instead of injecting an invalid remote +// adapter target. The reconciler already trims before publishing +// status.remoteStorage.endpoint, but the webhook trims defensively here too. // // For external storage with an empty/whitespace spec.remoteStorage.endpoint there is NO // fallback to status. The reconciler treats that state as @@ -672,7 +667,7 @@ func effectiveEndpoint(cache *cachev1alpha1.CacheBackend) string { if cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology != "" && cache.Status.RemoteStorage != nil { return strings.TrimSpace(cache.Status.RemoteStorage.Endpoint) } - return strings.TrimSpace(cache.Status.Endpoint) + return "" } // hasContainer reports whether containers already includes one named name. diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index e0983216..229b41f7 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -40,7 +40,7 @@ import ( const ( testVLLMEngineContainerName = "vllm" testSubscriberImage = "subscriber:test" - testEnvLMCacheRemoteURL = "LMCACHE_REMOTE_URL" + testRetiredLMCacheRemoteURL = "LMCACHE_REMOTE_URL" testEnvLMCacheChunkSize = "LMCACHE_CHUNK_SIZE" testEnvVLLMUseV1 = "VLLM_USE_V1" testEnvPythonHashSeed = "PYTHONHASHSEED" @@ -55,7 +55,6 @@ func newVLLMRegistry(configs ...builtinruntime.SubscriberConfig) *adapterruntime } registry := adapterruntime.NewRegistry() registry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(config)) - registry.Register(builtinruntime.NewVLLMLMCacheAdapter(config)) return registry } @@ -103,11 +102,12 @@ func referenceUpsertEnv(env []corev1.EnvVar, want corev1.EnvVar) []corev1.EnvVar return append(env, want) } -func externalLMCacheStorage(endpoint string) *cachev1alpha1.CacheBackendRemoteStorageSpec { +func externalRedisStorage(endpoint string) *cachev1alpha1.CacheBackendRemoteStorageSpec { return &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: endpoint, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } } @@ -220,14 +220,14 @@ func sglangEnginePod(name string, labels map[string]string) *corev1.Pod { Env: []corev1.EnvVar{ {Name: "USER_FLAG", Value: "preserved"}, }, - Args: []string{"--model-path", "Qwen/Qwen2.5-0.5B-Instruct"}, + Args: []string{"--model-path", "Qwen/Qwen2.5-0.5B-Instruct", "--page-size", "64"}, }}, }, } } -// readyCacheBackend returns a CacheBackend with status.endpoint published, -// a vLLM integration, and an EngineSelector keyed on a single label. +// readyCacheBackend returns a typed host-only CacheBackend with a vLLM +// integration and an EngineSelector keyed on a single label. // The metadata.uid is set to a stable fake so the webhook's // AnnotationInjectedByUID stamp has a value to compare against in tests // that assert the annotation contents (a real apiserver would assign one @@ -242,10 +242,21 @@ func readyCacheBackend(name, namespace string, selector map[string]string) *cach Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, L1Capacity: resource.MustParse("4Gi"), MaxWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("6Gi")}, + }, + }}, + }, RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, }, Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, @@ -253,7 +264,11 @@ func readyCacheBackend(name, namespace string, selector map[string]string) *cach EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: selector}, }, Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: name + ".cache-ns.svc.cluster.local:65432", + RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageStatus{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Endpoint: name + "." + namespace + ".svc.cluster.local:6379", + Ready: metav1.ConditionTrue, + }, }, } } @@ -286,43 +301,6 @@ func newHandlerWithSubscriber(t *testing.T, objs ...client.Object) *EngineInject } } -func TestHandle_MatchAndInject(t *testing.T) { - const ns = "engines" - cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) - h := newHandler(t, cb) - pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got: %+v", resp.Result) - } - if len(resp.Patches) == 0 { - t.Fatalf("expected JSON patches, got none") - } - - mutated := applyPatches(t, req.Object.Raw, resp) - mustHaveEnv(t, mutated, "USER_FLAG", "preserved") - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, - "lm://"+cb.Status.Endpoint) - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - if got, want := mutated.Annotations[AnnotationInjectedBy], ns+"/"+cb.Name; got != want { - t.Fatalf("annotation %s: got %q, want %q", AnnotationInjectedBy, got, want) - } - // Pin the webhook-only proof-of-injection annotation against the - // matched CR's UID. The engine-pod-events controller skips emission - // when this doesn't match; a regression in the success-path stamp - // would break the binding signal end-to-end. - if got, want := mutated.Annotations[AnnotationInjectedByUID], string(cb.UID); got != want { - t.Fatalf("annotation %s: got %q, want %q (matched CR UID)", AnnotationInjectedByUID, got, want) - } - if got, want := mutated.Annotations[AnnotationInjectedGeneration], fmt.Sprint(cb.Generation); got != want { - t.Fatalf("annotation %s: got %q, want %q", AnnotationInjectedGeneration, got, want) - } - mustHaveArgPair(t, mutated, "--model", "Qwen/Qwen2.5-0.5B-Instruct") - mustHaveArgFlag(t, mutated, "--kv-transfer-config") -} - func typedVLLMPodLocalBackend(name, namespace string, selector map[string]string) *cachev1alpha1.CacheBackend { cb := readyCacheBackend(name, namespace, selector) chunkSize := int32(256) @@ -341,7 +319,7 @@ func typedVLLMPodLocalBackend(name, namespace string, selector map[string]string }}, } cb.Spec.RemoteStorage = nil - cb.Status.Endpoint = "" + cb.Status.RemoteStorage = nil return cb } @@ -448,8 +426,7 @@ func TestHandle_MatchAndInject_SGLang(t *testing.T) { mustHaveEnv(t, mutated, "LMCACHE_USE_EXPERIMENTAL", "True") // Proof it went through the SGLang MP path: the vLLM-only connector arg/env - // must be absent, the old lm:// env must NOT be injected, and the MP-worker - // native sidecar must be present. + // must be absent and the common MP server native sidecar must be present. for _, c := range mutated.Spec.Containers { if c.Name != "sglang" { continue @@ -463,22 +440,16 @@ func TestHandle_MatchAndInject_SGLang(t *testing.T) { if e.Name == testEnvVLLMUseV1 || e.Name == testEnvPythonHashSeed { t.Fatalf("SGLang pod got vLLM-only env %q (SGLang injects neither)", e.Name) } - if e.Name == testEnvLMCacheRemoteURL { - t.Fatalf("SGLang MP wire must not inject %s", testEnvLMCacheRemoteURL) + if e.Name == testRetiredLMCacheRemoteURL { + t.Fatalf("SGLang MP wire must not inject retired %s", testRetiredLMCacheRemoteURL) } } } - hasWorker := false - for _, ic := range mutated.Spec.InitContainers { - if ic.Name == "lmcache-mp-worker" { - hasWorker = true - } - } - if !hasWorker { - t.Fatalf("MP-worker sidecar not injected; initContainers = %+v", mutated.Spec.InitContainers) + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") == nil { + t.Fatalf("MP server sidecar not injected; initContainers = %+v", mutated.Spec.InitContainers) } - if got := mutated.Labels[LabelLMCacheMPMetrics]; got != "" { - t.Fatalf("legacy topology-less SGLang pod got typed MP metrics label %s=%q", LabelLMCacheMPMetrics, got) + if got := mutated.Labels[LabelLMCacheMPMetrics]; got != "true" { + t.Fatalf("typed SGLang pod label %s=%q, want true", LabelLMCacheMPMetrics, got) } } @@ -508,7 +479,6 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { } h := newHandler(t, cb) pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) - pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--page-size=1") req := newRequest(t, pod, ns) resp := h.Handle(context.Background(), req) @@ -544,7 +514,7 @@ func TestHandle_TypedPodLocalSGLangUsesCommonMPServer(t *testing.T) { // the engine wire. The response message is the actionable admission trace; // controller status subsequently counts the Pod as uncovered. incompatible := sglangEnginePod("sg-engine-incompatible", map[string]string{"app": "sglang"}) - incompatible.Spec.Containers[0].Args = append(incompatible.Spec.Containers[0].Args, "--page-size=96") + incompatible.Spec.Containers[0].Args = []string{"--model-path", "Qwen/Qwen2.5-0.5B-Instruct", "--page-size=96"} incompatibleReq := newRequest(t, incompatible, ns) incompatibleResp := h.Handle(context.Background(), incompatibleReq) if !incompatibleResp.Allowed || len(incompatibleResp.Patches) != 0 { @@ -561,6 +531,7 @@ func TestHandle_MatchAndInject_SGLangHiCacheWithoutEndpoint(t *testing.T) { cb := readyCacheBackend("hicache", ns, map[string]string{"app": "sglang"}) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.LMCache = nil cb.Spec.RemoteStorage = nil cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{ @@ -569,7 +540,7 @@ func TestHandle_MatchAndInject_SGLangHiCacheWithoutEndpoint(t *testing.T) { IOBackend: cachev1alpha1.SGLangHiCacheIOKernel, MemoryLayout: cachev1alpha1.SGLangHiCacheMemoryPageFirst, } - cb.Status.Endpoint = "" + cb.Status.RemoteStorage = nil h := newHandler(t, cb) pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) @@ -604,6 +575,7 @@ func TestHandle_CanonicalSGLangHiCacheWithRemoteStorageFailsOpen(t *testing.T) { cb := readyCacheBackend("hicache-remote", ns, map[string]string{"app": "sglang"}) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.LMCache = nil cb.Spec.Runtime = "" cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"} cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ @@ -628,13 +600,14 @@ func TestHandle_SGLangHiCacheConflictFailsOpenWithoutPartialInjection(t *testing cb := readyCacheBackend("hicache", ns, map[string]string{"app": "sglang"}) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache + cb.Spec.LMCache = nil cb.Spec.RemoteStorage = nil cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{ Ratio: "2", WritePolicy: cachev1alpha1.SGLangHiCacheWriteThrough, } - cb.Status.Endpoint = "" + cb.Status.RemoteStorage = nil pod := sglangEnginePod("sg-engine-a", map[string]string{"app": "sglang"}) pod.Spec.Containers[0].Args = append(pod.Spec.Containers[0].Args, "--hicache-ratio=3") @@ -648,232 +621,11 @@ func TestHandle_SGLangHiCacheConflictFailsOpenWithoutPartialInjection(t *testing } } -func TestHandle_MooncakeBackend_InjectsMooncakeStoreEndpoint(t *testing.T) { - // End-to-end pod-webhook path for a managed Mooncake backend: the handler - // lists the CacheBackend, the built-in shipping registry selects the - // vLLM+LMCache adapter with a Mooncake binding, and the engine container is wired to the - // Mooncake master via the LMCache connector with the mooncakestore:// - // scheme (the lm:// analog) — plus the kvevent-subscriber sidecar. This is - // the advertised integration path; the adapter-level tests don't exercise - // the webhook's registry selection + injection together. - const ns = "engines" - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{ - Name: "mc", - Namespace: ns, - UID: types.UID("cb-mc-uid"), - }, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "Qwen/Qwen2.5-0.5B-Instruct"}, - }, - // Mooncake status.endpoint is the master's RPC host:port (the - // reconciler publishes the Service's first port, 50051). - Status: cachev1alpha1.CacheBackendStatus{ - Endpoint: "mc.engines.svc.cluster.local:50051", - }, - } - h := newHandlerWithSubscriber(t, cb) - pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed || len(resp.Patches) == 0 { - t.Fatalf("expected Allowed with patches; Allowed=%v patches=%d result=%+v", resp.Allowed, len(resp.Patches), resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - - // The defining difference from the LMCache path: the remote URL carries - // the mooncakestore:// scheme, pointed at the master RPC endpoint. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "mooncakestore://"+cb.Status.Endpoint) - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - mustHaveArgFlag(t, mutated, "--kv-transfer-config") - // User-set engine arg survives the merge (merge, not clobber). - mustHaveArgPair(t, mutated, "--model", "Qwen/Qwen2.5-0.5B-Instruct") - if got, want := mutated.Annotations[AnnotationInjectedBy], ns+"/"+cb.Name; got != want { - t.Fatalf("annotation %s: got %q, want %q", AnnotationInjectedBy, got, want) - } - - // The kvevent-subscriber sidecar is appended on the Mooncake path too - // (same shared builder; vLLM's KV-event stream is store-independent). - sub := findContainer(mutated, enginebinding.SubscriberContainerName) - if sub == nil { - t.Fatalf("subscriber sidecar missing on Mooncake path; containers = %v", containerNames(mutated)) - } - if !argPresent(sub.Args, "--hash-scheme=vllm") { - t.Fatalf("subscriber must tag events hash-scheme=vllm; args = %v", sub.Args) - } - if !argPresent(sub.Args, "--engine-metrics-url=http://127.0.0.1:8000/metrics") { - t.Fatalf("vLLM subscriber must scrape :8000/metrics; args = %v", sub.Args) - } - if !argPresent(sub.Args, "--model-id=Qwen/Qwen2.5-0.5B-Instruct") { - t.Fatalf("subscriber --model-id derived from observation.modelID missing; args = %v", sub.Args) - } -} - -func TestHandle_MooncakeBackend_EngineHostNetworkIsOptIn(t *testing.T) { - // Mooncake's mesh is dialed FROM the engine at a node IP on a negotiated - // port, so an overlay engine pod transfers zero KV while the backend reports - // Ready. spec.integration.engineHostNetwork moves matched engine pods onto - // the host network — but only when the operator asks for it: hostNetwork is - // a privilege, and mutating webhooks run BEFORE Pod Security validation, so - // injecting it unasked would turn a working pod into one a "restricted" - // namespace rejects, blaming Pod Security rather than this controller. - // - // The adapter unit test pins InjectEngineConfig; this pins the whole - // admission path, which is the surface the operator's pod actually crosses. - const ns = "engines" - backend := func(optIn bool) *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "mc", Namespace: ns, UID: types.UID("cb-mc-uid")}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - EngineHostNetwork: optIn, - }, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: "mc.engines.svc.cluster.local:50051"}, - } - } - - t.Run("NotInjectedByDefault", func(t *testing.T) { - h := newHandlerWithSubscriber(t, backend(false)) - req := newRequest(t, vllmEnginePod("engine-a", map[string]string{"app": "vllm"}), ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed; result=%+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - - if mutated.Spec.HostNetwork { - t.Fatal("webhook moved an engine pod onto the host network without spec.integration.engineHostNetwork; " + - "that silently escalates the pod's privileges and Pod Security would reject it downstream") - } - if mutated.Spec.DNSPolicy != "" { - t.Fatalf("dnsPolicy rewritten to %q without the opt-in; want it left to the cluster default", mutated.Spec.DNSPolicy) - } - // The rest of the Mooncake wiring still lands — the opt-in gates the - // networking rewrite only, never the connector env. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "mooncakestore://mc.engines.svc.cluster.local:50051") - }) - - t.Run("InjectedWhenOperatorOptsIn", func(t *testing.T) { - h := newHandlerWithSubscriber(t, backend(true)) - req := newRequest(t, vllmEnginePod("engine-a", map[string]string{"app": "vllm"}), ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed; result=%+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - - if !mutated.Spec.HostNetwork { - t.Fatal("engineHostNetwork=true but the pod stayed on the overlay; the engine cannot reach the Mooncake mesh") - } - // Without this the pod loses cluster DNS, and status.endpoint is a - // Service DNS name — the engine would fail to resolve the master. - if got, want := mutated.Spec.DNSPolicy, corev1.DNSClusterFirstWithHostNet; got != want { - t.Fatalf("dnsPolicy: got %q, want %q (hostNetwork pods lose cluster DNS without it, and status.endpoint is a DNS name)", got, want) - } - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "mooncakestore://mc.engines.svc.cluster.local:50051") - }) -} - -func TestHandle_MooncakeBackend_HostNetworkNeverGrantedToAnUnwiredPod(t *testing.T) { - // The host-network mutation lives INSIDE InjectEngineConfig, behind the same - // endpoint gate as the KV connector. So when the reconciler has not yet - // published status.endpoint, the pod admits fail-open with neither the - // connector nor hostNetwork — the two always travel together. - // - // That coupling is deliberate, and this test exists to keep it. Hoisting the - // hostNetwork mutation above the endpoint gate would grant a pod a privilege - // (host networking, which Pod Security gates) while giving it nothing to use - // the privilege for: without LMCACHE_REMOTE_URL the engine never dials the - // Mooncake mesh. The pod must be rolled once the backend reports Ready - // regardless — pods are immutable, and it needs the connector env either way. - const ns = "engines" - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "mc", Namespace: ns, UID: types.UID("cb-mc-uid")}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - EngineHostNetwork: true, - }, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - // The reconciler has not published the master's RPC address yet. - Status: cachev1alpha1.CacheBackendStatus{Endpoint: ""}, - } - h := newHandlerWithSubscriber(t, cb) - req := newRequest(t, vllmEnginePod("engine-a", map[string]string{"app": "vllm"}), ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("endpoint-not-ready must fail open, not reject the pod; result=%+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - - if mutated.Spec.HostNetwork { - t.Fatal("pod was granted hostNetwork while the backend had no endpoint — a Pod-Security-gated privilege " + - "handed to a pod with no KV connector to use it") - } - for _, c := range mutated.Spec.Containers { - for _, e := range c.Env { - if e.Name == testEnvLMCacheRemoteURL { - t.Fatalf("connector env %s injected without an endpoint", e.Name) - } - } - } -} - func TestHandle_LMCacheBackend_NeverMovesEnginePodOntoHostNetwork(t *testing.T) { - // The default path must stay on the overlay. hostNetwork is Mooncake's - // carve-out; a regression that leaked it into the LMCache adapter would - // break every "restricted"-PSA namespace running the shipping default. + // The typed MP path must stay on the overlay; moving the engine to the host + // network would break restricted Pod Security namespaces. const ns = "engines" - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "lm", Namespace: ns, UID: types.UID("cb-lm-uid")}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: "lm.engines.svc.cluster.local:8000"}, - } + cb := readyCacheBackend("lm", ns, map[string]string{"app": "vllm"}) h := newHandlerWithSubscriber(t, cb) req := newRequest(t, vllmEnginePod("engine-a", map[string]string{"app": "vllm"}), ns) @@ -926,9 +678,14 @@ func TestHandle_AppendsObservationSidecar(t *testing.T) { if !argPresent(sub.Args, "--tenant-id=$(POD_NAMESPACE)") { t.Fatalf("--tenant-id MUST use downward-API POD_NAMESPACE; args = %v", sub.Args) } - // The engine container is still wired with LMCache env — appending the - // sidecar must not regress the engine-side injection. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + if !argPresent(sub.Args, "--engine-metrics-url=http://127.0.0.1:8000/metrics") { + t.Fatalf("vLLM subscriber must scrape :8000/metrics; args = %v", sub.Args) + } + // Appending the observation sidecar must not regress typed MP injection. + mustHaveArgFlag(t, mutated, "--kv-transfer-config") + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") == nil { + t.Fatal("LMCache MP server missing after subscriber injection") + } } func TestHandle_AppendsObservationSidecar_SGLang(t *testing.T) { @@ -975,7 +732,7 @@ func TestHandle_AppendsObservationSidecar_SGLang(t *testing.T) { t.Fatalf("SGLang subscriber MUST tag --hash-scheme=sglang; args = %v", sub.Args) } if !argPresent(sub.Args, "--engine-metrics-url=http://127.0.0.1:30000/metrics") { - t.Fatalf("SGLang subscriber must scrape :30000/metrics (not vLLM's :8000); args = %v", sub.Args) + t.Fatalf("SGLang subscriber must scrape :30000/metrics; args = %v", sub.Args) } if !argPresent(sub.Args, "--model-id=Qwen/Qwen2.5-0.5B-Instruct") { t.Fatalf("--model-id derived from cb.spec.observation.modelID missing; args = %v", sub.Args) @@ -1011,9 +768,9 @@ func TestHandle_SidecarAppendIsIdempotent(t *testing.T) { // eventsOnlyCacheBackend returns an events-only (tier-1 routing) LMCache // CacheBackend: type=LMCache, spec.integration.mode=EventsOnly, a served model -// id (so ObservationSidecar emits a container), an engineSelector, and NO -// status.endpoint. It provisions no server, so the absent endpoint is the -// expected steady state — not a not-yet-reconciled race. +// id (so ObservationSidecar emits a container), an engineSelector, and no +// remote storage. The absent endpoint is the expected steady state, not a +// not-yet-reconciled race. func eventsOnlyCacheBackend(name, namespace string, selector map[string]string) *cachev1alpha1.CacheBackend { return &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{ @@ -1031,13 +788,12 @@ func eventsOnlyCacheBackend(name, namespace string, selector map[string]string) EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: selector}, Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "Qwen/Qwen2.5-0.5B-Instruct"}, }, - // No Status.Endpoint — events-only provisions no server, so the - // reconciler leaves it empty. The webhook MUST inject anyway. + // No remote-storage status: the webhook must inject the subscriber anyway. } } func TestHandle_EventsOnly_EmptyEndpoint_InjectsSubscriberWithoutConnector(t *testing.T) { - // An events-only backend has an EMPTY status.endpoint by design (no + // An events-only backend has no status.remoteStorage by design (no // provisioned server), but it must NOT fail-open the way a managed backend // with a not-yet-published endpoint does. The webhook injects: the pod is // patched, the kvevent-subscriber sidecar is appended, the injected-by @@ -1054,7 +810,7 @@ func TestHandle_EventsOnly_EmptyEndpoint_InjectsSubscriberWithoutConnector(t *te t.Fatalf("expected Allowed, got: %+v", resp.Result) } if len(resp.Patches) == 0 { - t.Fatalf("events-only backend with empty status.endpoint must INJECT, not fail-open; got no patches") + t.Fatalf("events-only backend with no remote-storage endpoint must inject, not fail open; got no patches") } mutated := applyPatches(t, req.Object.Raw, resp) @@ -1107,15 +863,20 @@ func TestHandle_EventsOnly_EmptyEndpoint_InjectsSubscriberWithoutConnector(t *te func TestHandle_OffloadManagedBackend_EmptyEndpoint_FailsOpen(t *testing.T) { // Contrast with the events-only case above: an Offload (default-mode) - // managed backend whose status.endpoint is not yet published MUST fail-open + // managed backend whose status.remoteStorage.endpoint is not yet published must fail open // — admit unmodified, no subscriber sidecar, no injected-by annotation — // because the connector it would wire needs a real dial target. This pins // that the events-only inject path is mode-gated, not a blanket // "inject on empty endpoint". const ns = "engines" cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "Qwen/Qwen2.5-0.5B-Instruct"} - cb.Status.Endpoint = "" // Offload mode, reconciler hasn't published yet. + cb.Status.RemoteStorage = nil h := newHandlerWithSubscriber(t, cb) pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) req := newRequest(t, pod, ns) @@ -1125,7 +886,7 @@ func TestHandle_OffloadManagedBackend_EmptyEndpoint_FailsOpen(t *testing.T) { t.Fatalf("expected Allowed (fail-open), got: %+v", resp.Result) } if len(resp.Patches) != 0 { - t.Fatalf("Offload managed backend with empty status.endpoint must fail-open (no patches), got %d: %+v", + t.Fatalf("Offload managed backend with empty status.remoteStorage.endpoint must fail open (no patches), got %d: %+v", len(resp.Patches), resp.Patches) } // Fail-open never stamps injected-by; the inbound pod carried none, so a @@ -1327,351 +1088,21 @@ func TestHandle_EventsOnly_EngineOverrides_DoNotTouchEngineContainer(t *testing. } } -func TestHandle_ExternalBackend_InjectsOperatorEndpoint(t *testing.T) { - // A pod that matches an externally owned CR's engine selector must come out - // of admission wired to the operator-supplied endpoint via the - // LMCache engine wire format — the controller doesn't render a - // Service for the cache, so the only source of truth for the - // address is spec.remoteStorage.endpoint (mirrored to status.endpoint by - // reconcileExternal). - const ( - ns = "engines" - endpoint = "external-cache.example:8200" - ) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(endpoint), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: endpoint}, - } - - s := newScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() - reg := newVLLMRegistry() - h := &EngineInjector{Reader: c, Registry: reg, Log: logr.Discard()} - - pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got %+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - - // LMCACHE_REMOTE_URL must be the operator-supplied endpoint with the - // lm:// scheme prepended, identical to what the managed adapter - // would write for the same endpoint. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+endpoint) - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - // User --model arg survives the merge — the adapter only adds; it - // never clobbers user-set args. - if !containsArgPairLocal(mutated.Spec.Containers[0].Args, "--model", "Qwen/Qwen2.5-0.5B-Instruct") { - t.Fatalf("user --model arg was lost; args = %v", mutated.Spec.Containers[0].Args) - } - // The external-ownership path attaches no observation sidecar — the - // controller has no observability seam into an operator-managed cache. - if c := findContainer(mutated, enginebinding.SubscriberContainerName); c != nil { - t.Fatalf("External backend must NOT get a subscriber sidecar; found %+v", c) - } - if mutated.Annotations[AnnotationInjectedBy] != ns+"/ext" { - t.Fatalf("annotation %s = %q, want %q", - AnnotationInjectedBy, mutated.Annotations[AnnotationInjectedBy], ns+"/ext") - } -} - -func TestHandle_ExternalBackend_InvalidSpecEndpoint_FailsOpen(t *testing.T) { - // An externally owned CR carrying a malformed spec.remoteStorage.endpoint - // must not be wired — - // injecting LMCACHE_REMOTE_URL=lm://https://... or lm://2001:db8::1 - // would crash the engine at startup. effectiveEndpoint applies the - // same shape check the admission webhook uses and returns "" for - // invalid values, so the existing fail-open branch admits the pod - // un-wired and the operator sees the shape error in the response - // reason instead of an engine-pod crash log. - const ns = "engines" - for _, tc := range []struct { - name, endpoint string - }{ - {"bad-scheme", "https://cache.example.com:443/api"}, - {"portless-host", "cache.example.com"}, - {"non-numeric-port", "cache.example.com:not-a-port"}, - {"zero-port", "cache.example.com:0"}, - {"out-of-range-port", "cache.example.com:70000"}, - {"embedded-whitespace", "cache example:8200"}, - } { - t.Run(tc.name, func(t *testing.T) { - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-bad", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(tc.endpoint), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - } - s := newScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() - reg := newVLLMRegistry() - h := &EngineInjector{Reader: c, Registry: reg, Log: logr.Discard()} - - pod := vllmEnginePod("engine", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed (fail-open), got %+v", resp.Result) - } - // Zero patches — no injection happened. - if len(resp.Patches) != 0 { - t.Fatalf("expected no patches on invalid endpoint; got %d: %v", len(resp.Patches), resp.Patches) - } - // Response message must name the canonical spec field, not status.endpoint. - if msg := resp.Result.Message; !strings.Contains(msg, "spec.remoteStorage.endpoint") { - t.Fatalf("fail-open reason should mention spec.remoteStorage.endpoint for External; got %q", msg) - } - }) - } -} - -func TestEffectiveEndpointCanonicalExternalUsesProviderProtocol(t *testing.T) { - cache := &cachev1alpha1.CacheBackend{ - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "mooncakestore://cache.example:50051", - }, - }, - } - if got := effectiveEndpoint(cache); got != cache.Spec.RemoteStorage.Endpoint { - t.Fatalf("effectiveEndpoint(Mooncake) = %q, want %q", got, cache.Spec.RemoteStorage.Endpoint) - } - - cache.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cache.Spec.RemoteStorage.Provider = cachev1alpha1.CacheBackendRemoteStorageProviderRedis - cache.Spec.RemoteStorage.Endpoint = "lm://redis.example:6379" - if got := effectiveEndpoint(cache); got != "" { - t.Fatalf("effectiveEndpoint(Redis with lm scheme) = %q, want empty fail-open endpoint", got) - } -} - -func TestHandle_ExternalBackend_StatusEmpty_UsesSpecDirectly(t *testing.T) { - // Pod admission is CREATE-only — if an engine pod admits before the - // controller has mirrored spec.remoteStorage.endpoint into status.endpoint, - // the webhook would fail-open and leave the pod unwired *forever* (no - // re-admission on subsequent status updates). For externally owned CRs the - // webhook sources the endpoint from spec.remoteStorage.endpoint directly - // (NOT "falling back" — effectiveEndpoint ownership-scopes the source so - // external ownership never reads status.endpoint, preventing wiring against a - // stale mirror during an endpoint update). Without this, applying - // the externally owned CacheBackend and the engine Deployment in the same - // kubectl apply silently produces unwired engine pods. - const ( - ns = "engines" - endpoint = "external-cache.example:8200" - ) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(endpoint), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - // Deliberately no Status: simulates the race where pod admission - // fires before reconcileExternal has patched status.endpoint. - } - - s := newScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() - reg := newVLLMRegistry() - h := &EngineInjector{Reader: c, Registry: reg, Log: logr.Discard()} - - pod := vllmEnginePod("engine-race", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got %+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+endpoint) -} - -func TestHandle_ExternalBackend_PrefersSpecOverStaleStatus(t *testing.T) { - // When the operator updates spec.remoteStorage.endpoint for an externally - // owned CR but a new engine pod admits before the reconciler patches status, - // the - // pod must be wired to the NEW spec.remoteStorage.endpoint — not the stale - // status.endpoint. Pod admission is CREATE-only, so a pod wired to - // the old address on admission stays misrouted forever. - const ( - ns = "engines" - freshSpec = "new-cache.example:8200" - staleStatus = "old-cache.example:8200" - ) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(freshSpec), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: staleStatus}, - } - - s := newScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() - reg := newVLLMRegistry() - h := &EngineInjector{Reader: c, Registry: reg, Log: logr.Discard()} - - pod := vllmEnginePod("engine-stale", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got %+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - // Must use spec.remoteStorage.endpoint, NOT the stale status.endpoint. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+freshSpec) - for _, e := range mutated.Spec.Containers[0].Env { - if e.Name == testEnvLMCacheRemoteURL && e.Value == "lm://"+staleStatus { - t.Fatalf("pod wired to stale status.endpoint %q; should be spec.remoteStorage.endpoint %q", staleStatus, freshSpec) - } - } -} - -func TestHandle_ExternalBackend_UpperCaseSchemeNormalised(t *testing.T) { - // Admission lowercases the scheme during shape validation, so - // `LM://cache.example:8200` admits. The pod webhook must then - // normalise to lower-case `lm://` at injection — passing the - // operator-typed value through verbatim would produce - // `LMCACHE_REMOTE_URL=lm://LM://cache.example:8200`, a double- - // prefix the engine connector rejects. - const ( - ns = "engines" - operatorTyped = "LM://cache.example.com:8200" - ) - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "ext-up", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(operatorTyped), - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - } - - s := newScheme(t) - c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() - reg := newVLLMRegistry() - h := &EngineInjector{Reader: c, Registry: reg, Log: logr.Discard()} - - pod := vllmEnginePod("engine-up", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got %+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - // Must be the canonical lower-case scheme, with the original - // host portion preserved verbatim. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://cache.example.com:8200") -} - -func TestHandle_WhitespaceStatusEndpointFailsOpen(t *testing.T) { - // A CR that predates the trim-in-reconciler change could carry a - // whitespace-only status.endpoint. The webhook MUST treat that as - // missing rather than injecting `LMCACHE_REMOTE_URL=lm:// ` which - // the engine connector would reject at runtime. The defensive trim - // applies to whichever field effectiveEndpoint reads for the CR's - // ownership — spec.remoteStorage.endpoint for external ownership (which - // never reads status), and status.endpoint for managed ownership. - const ns = "engines" - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "managed-ws", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: " "}, - } - - h := newHandler(t, cb) - pod := vllmEnginePod("engine-ws", map[string]string{"app": "vllm"}) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got %+v", resp.Result) - } - mutated := applyPatches(t, req.Object.Raw, resp) - for _, e := range mutated.Spec.Containers[0].Env { - if e.Name == testEnvLMCacheRemoteURL { - t.Fatalf("whitespace status.endpoint must not become injected env; got %s=%q", e.Name, e.Value) - } - } -} - func TestHandle_ManagedBackend_StatusEmpty_FailsOpen(t *testing.T) { // Counterpart to the external-ownership path: managed backends MUST wait - // for status.endpoint (the reconciler builds it from the rendered + // for status.remoteStorage.endpoint (the reconciler builds it from the rendered // Service). spec.remoteStorage.endpoint is admission-rejected for managed // ownership, so there's nothing else to fall back on — the webhook must // fail-open without injecting until status catches up. const ns = "engines" - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: ns}, - Spec: cachev1alpha1.CacheBackendSpec{ - Type: cachev1alpha1.CacheBackendTypeLMCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "vllm"}, - }, - }, - // No Status.Endpoint published yet. + cb := readyCacheBackend("managed", ns, map[string]string{"app": "vllm"}) + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } + cb.Status.RemoteStorage = nil + // No status.remoteStorage.endpoint has been published yet. h := newHandler(t, cb) pod := vllmEnginePod("engine-managed", map[string]string{"app": "vllm"}) @@ -1681,31 +1112,14 @@ func TestHandle_ManagedBackend_StatusEmpty_FailsOpen(t *testing.T) { if !resp.Allowed { t.Fatalf("expected Allowed, got %+v", resp.Result) } - // The pod must NOT have LMCACHE_REMOTE_URL because there's no - // endpoint to wire it to — the fallback is External-only. + // The pod must remain entirely unmodified because the managed endpoint has + // not been observed yet. mutated := applyPatches(t, req.Object.Raw, resp) - if len(mutated.Spec.Containers) == 0 { - t.Fatalf("pod has no containers after admission") - } - for _, e := range mutated.Spec.Containers[0].Env { - if e.Name == testEnvLMCacheRemoteURL { - t.Fatalf("managed CR with no status.endpoint must NOT trigger injection; got %s=%q", e.Name, e.Value) - } + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") != nil || containsArgFlag(mutated.Spec.Containers[0].Args, "--kv-transfer-config") { + t.Fatalf("managed CR with no status.remoteStorage.endpoint unexpectedly injected MP wiring: %+v", mutated.Spec) } } -// containsArgPairLocal mirrors the helper in envtest_integration_test.go; -// the two test files don't share state (envtest skips without -// KUBEBUILDER_ASSETS) so each file has its own copy. -func containsArgPairLocal(args []string, flag, value string) bool { - for i := 0; i < len(args)-1; i++ { - if args[i] == flag && args[i+1] == value { - return true - } - } - return false -} - func TestHandle_ExternalBackend_NoSidecar(t *testing.T) { // Negative case: a CacheBackend matched by a // runtime whose adapter returns no sidecar (the reference adapter here, @@ -1714,6 +1128,9 @@ func TestHandle_ExternalBackend_NoSidecar(t *testing.T) { const ns = "engines" cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntime(testRuntimeReference) + cb.Spec.Type = cachev1alpha1.CacheBackendType("reference") + cb.Spec.LMCache = nil + cb.Spec.RemoteStorage = externalRedisStorage("redis.example:6379") s := newScheme(t) c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() reg := adapterruntime.NewRegistry() @@ -1753,7 +1170,7 @@ func TestHandle_SidecarOptInDefaultsToNoSidecar(t *testing.T) { if c := findContainer(mutated, enginebinding.SubscriberContainerName); c != nil { t.Fatalf("default install must NOT auto-attach the sidecar; got %+v", c) } - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + mustHaveArgFlag(t, mutated, "--kv-transfer-config") } func TestHandle_SidecarSkippedWithoutModel(t *testing.T) { @@ -1774,7 +1191,7 @@ func TestHandle_SidecarSkippedWithoutModel(t *testing.T) { if c := findContainer(mutated, enginebinding.SubscriberContainerName); c != nil { t.Fatalf("sidecar must be skipped without a model id; got %+v", c) } - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + mustHaveArgFlag(t, mutated, "--kv-transfer-config") } func TestHandle_SidecarErrorIsFailOpen(t *testing.T) { @@ -1785,6 +1202,9 @@ func TestHandle_SidecarErrorIsFailOpen(t *testing.T) { cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) cb.Spec.Runtime = "" cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntime("stub-fail") + cb.Spec.Type = cachev1alpha1.CacheBackendType("reference") + cb.Spec.LMCache = nil + cb.Spec.RemoteStorage = externalRedisStorage("redis.example:6379") s := newScheme(t) c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() reg := adapterruntime.NewRegistry() @@ -1947,39 +1367,10 @@ func TestHandle_FullyInjected_NoOpPatch(t *testing.T) { } } -func TestHandle_PartialEnvOnly_StillConverges(t *testing.T) { - // Regression for the round-5 Codex finding: a pod that already carries - // LMCACHE_REMOTE_URL but is missing the rest of the contract (no - // VLLM_USE_V1, no --kv-transfer-config arg) MUST still get the - // remaining wiring filled in. A lenient env-presence short-circuit - // would leave the pod permanently misconfigured; the adapter is the - // source of truth and we always call it. - const ns = "engines" - cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) - h := newHandler(t, cb) - pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) - pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{ - Name: testEnvLMCacheRemoteURL, - Value: "lm://stale.example:65432", - }) - req := newRequest(t, pod, ns) - - resp := h.Handle(context.Background(), req) - if !resp.Allowed || len(resp.Patches) == 0 { - t.Fatalf("partial-wired pod must still get the missing fields; Allowed=%v patches=%d", resp.Allowed, len(resp.Patches)) - } - mutated := applyPatches(t, req.Object.Raw, resp) - // The stale URL is overwritten with the canonical one for the matched - // backend, and the missing pieces are added. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - mustHaveArgFlag(t, mutated, "--kv-transfer-config") -} - func TestHandle_EndpointNotPublished_FailOpen(t *testing.T) { const ns = "engines" cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) - cb.Status.Endpoint = "" + cb.Status.RemoteStorage = nil h := newHandler(t, cb) pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) req := newRequest(t, pod, ns) @@ -2199,6 +1590,9 @@ func TestHandle_RegistryOverride_UsedInsteadOfDefault(t *testing.T) { cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) cb.Spec.Runtime = "" cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntime(testRuntimeReference) + cb.Spec.Type = cachev1alpha1.CacheBackendType("reference") + cb.Spec.LMCache = nil + cb.Spec.RemoteStorage = externalRedisStorage("redis.example:6379") s := newScheme(t) c := fake.NewClientBuilder().WithScheme(s).WithObjects(cb).Build() reg := adapterruntime.NewRegistry() @@ -2212,7 +1606,7 @@ func TestHandle_RegistryOverride_UsedInsteadOfDefault(t *testing.T) { t.Fatalf("expected Allowed with patches; got Allowed=%v patches=%d", resp.Allowed, len(resp.Patches)) } mutated := applyPatches(t, req.Object.Raw, resp) - mustHaveEnv(t, mutated, testReferenceCacheEndpoint, cb.Status.Endpoint) + mustHaveEnv(t, mutated, testReferenceCacheEndpoint, cb.Spec.RemoteStorage.Endpoint) } func TestHandle_PodNamespaceDefaultedFromRequest(t *testing.T) { @@ -2390,9 +1784,7 @@ func TestHandle_EngineOverrides_EnvUpsertAndArgAppend(t *testing.T) { Args: []string{"--max-model-len", "8192"}, Env: []corev1.EnvVar{ {Name: "FOO", Value: "bar"}, - // Override a tunable canonical env value, which is allowed - // because LMCACHE_CHUNK_SIZE is NOT reserved. - {Name: testEnvLMCacheChunkSize, Value: "512"}, + {Name: "EXTRA_TUNABLE", Value: "512"}, }, } h := newHandler(t, cb) @@ -2407,12 +1799,9 @@ func TestHandle_EngineOverrides_EnvUpsertAndArgAppend(t *testing.T) { // New env appended. mustHaveEnv(t, mutated, "FOO", "bar") - // Override wins for the tunable name (LMCACHE_CHUNK_SIZE is an - // adapter-owned canonical entry — the override surface can touch it). - mustHaveEnv(t, mutated, testEnvLMCacheChunkSize, "512") - // Canonical reserved env still landed unchanged. - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + mustHaveEnv(t, mutated, "EXTRA_TUNABLE", "512") + // Canonical typed-MP env still lands unchanged. + mustHaveEnv(t, mutated, testEnvPythonHashSeed, "0") // User-template env preserved. mustHaveEnv(t, mutated, "USER_FLAG", "preserved") @@ -2457,7 +1846,7 @@ func TestHandle_EngineOverrides_DoNotMutateUserTemplate(t *testing.T) { // User-owned env untouched by the CR-driven override + suppress. mustHaveEnv(t, mutated, "USER_FLAG", "preserved") // Canonical injection still landed. - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") + mustHaveEnv(t, mutated, testEnvPythonHashSeed, "0") mustHaveArgFlag(t, mutated, "--kv-transfer-config") } @@ -2495,8 +1884,7 @@ func TestHandle_EngineOverrides_NoOverride_ByteIdenticalToBaseline(t *testing.T) mutated := applyPatches(t, req.Object.Raw, resp) // Sanity: canonical injection lands as expected — so a green test // is meaningful (not green by producing an empty patch set). - mustHaveEnv(t, mutated, testEnvVLLMUseV1, "1") - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + mustHaveEnv(t, mutated, testEnvPythonHashSeed, "0") mustHaveArgFlag(t, mutated, "--kv-transfer-config") raw, err := json.Marshal(mutated) @@ -2536,7 +1924,7 @@ func TestHandle_FailOpenClearsForgedInjectedByAnnotation(t *testing.T) { var h *EngineInjector if tc.seedCB { cb := readyCacheBackend("primary", ns, map[string]string{"app": "vllm"}) - cb.Status.Endpoint = "" // force the endpoint-not-published fail-open path + cb.Status.RemoteStorage = nil // exercise fail-open while optional Redis is unavailable h = newHandler(t, cb) } else { h = newHandler(t) @@ -2620,8 +2008,8 @@ func TestHandle_KernelCheckInitContainer_AppendedOnGPUPod(t *testing.T) { t.Fatalf("kernel-check init container %q missing from Spec.InitContainers; got: %v", enginebinding.LMCacheKernelCheckContainerName, initContainerNames(mutated)) } - // Engine-side injection must still have landed. - mustHaveEnv(t, mutated, testEnvLMCacheRemoteURL, "lm://"+cb.Status.Endpoint) + // Engine-side typed MP injection must still have landed. + mustHaveArgFlag(t, mutated, "--kv-transfer-config") } // TestHandle_KernelCheckInitContainer_Idempotent verifies that a second @@ -2919,7 +2307,7 @@ func TestHandle_EventsOnlyExternal_NoConnectorWiring(t *testing.T) { Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, - RemoteStorage: externalLMCacheStorage(endpoint), + RemoteStorage: externalRedisStorage(endpoint), Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, @@ -2929,7 +2317,6 @@ func TestHandle_EventsOnlyExternal_NoConnectorWiring(t *testing.T) { }, Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "Qwen/Qwen2.5-0.5B-Instruct"}, }, - Status: cachev1alpha1.CacheBackendStatus{Endpoint: endpoint}, } s := newScheme(t) diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter.go b/internal/webhook/v1alpha1/cachebackend_defaulter.go index aa274bcc..bb31396c 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter.go @@ -15,8 +15,7 @@ import ( // Phase-1 defaults applied by the mutating webhook. Centralised here so the // tests pin the same constants the handler uses. // -// Literal-value defaults (spec.type=LMCache, spec.deploymentKind=Deployment, -// spec.replicas=1, spec.integration.mode=Offload, +// Literal-value defaults (spec.type=LMCache, spec.integration.mode=Offload, // spec.integration.role=ReadWrite, spec.integration.failOpen=true) are // expressed via `+kubebuilder:default=` markers on the API types and stamped // by the apiserver before this webhook runs. The webhook handles @@ -27,9 +26,6 @@ import ( // operator omits observation entirely the webhook materialises it here so // the persisted CR carries the readiness-gate deadline rather than relying // on the controller's runtime fallback. -// - spec.autoscaling.minReplicas: cluster-context default computed from -// spec.replicas at admission so the HPA's floor matches the operator's -// baseline declaration rather than a hard-coded constant. // // Per-field rationale lives in the godoc on each spec field; this comment // is the index for the webhook-stamped defaults specifically. @@ -42,8 +38,8 @@ const ( // CacheBackendDefaulter applies the Phase-1 defaults that CRD-schema // `+kubebuilder:default=` markers cannot express at admission time. Literal -// defaults (spec.type, deploymentKind, replicas, -// integration.mode, integration.role, integration.failOpen) ride on schema +// defaults (spec.type, integration.mode, integration.role, +// integration.failOpen) ride on schema // markers and are stamped by the apiserver before this handler runs; // the webhook handles context-sensitive and schema-inexpressible defaults: // @@ -51,10 +47,6 @@ const ( // - Materialises spec.observation to persist // spec.observation.firstEventTimeout when the operator omits the parent // block entirely. -// - Computes spec.autoscaling.minReplicas from spec.replicas when -// autoscaling is opted into and minReplicas is left unset — the HPA -// floor needs to follow the workload's baseline declaration, which is -// cluster-context the schema cannot encode. // // It does NOT stamp spec.integration.failOpen explicitly — once the // defaulter materialises spec.integration above, the apiserver applies @@ -76,11 +68,8 @@ type CacheBackendDefaulter struct{} // are applied. // - Materialises spec.observation when omitted so // spec.observation.firstEventTimeout carries the readiness-gate deadline. -// - Computes spec.autoscaling.minReplicas from spec.replicas when -// autoscaling is opted in and minReplicas is left unset. // -// Every other Phase-1 default (spec.type=LMCache, deploymentKind=Deployment, -// replicas=1, integration.mode=Offload, +// Every other default (spec.type=LMCache, integration.mode=Offload, // // integration.role=ReadWrite, // @@ -110,30 +99,5 @@ func (d *CacheBackendDefaulter) Default(ctx context.Context, cb *cachev1alpha1.C cb.Spec.Observation.FirstEventTimeout = &metav1.Duration{Duration: defaultFirstEventTimeout} } - // autoscaling.minReplicas defaults to spec.replicas when autoscaling is - // opted into and the operator left the floor unset. The literal - // spec.replicas default (=1) is applied by the apiserver from the - // `+kubebuilder:default` marker before this handler runs, so reading - // cb.Spec.Replicas here sees either the operator's explicit value or - // the schema default — never nil for a CR that came through admission. - // The nil guard is defence-in-depth for tests that construct a - // CacheBackend directly and call Default without the apiserver in the - // loop; we leave minReplicas alone in that case rather than dereference - // a nil pointer. - // - // The `>= 1` guard mirrors the CRD schema's `minimum: 1` on - // autoscaling.minReplicas: spec.replicas allows 0 (scale-to-zero), so a - // CR with `replicas: 0` + opted-in autoscaling would otherwise have the - // defaulter stamp `minReplicas: 0`, which the apiserver then rejects - // against the schema's minimum. Refusing to default in that case leaves - // the field unset so the operator's misconfiguration surfaces as a - // missing-required-field validation error against autoscaling rather - // than a webhook-introduced schema violation. - if cb.Spec.Autoscaling != nil && cb.Spec.Autoscaling.MinReplicas == nil && - cb.Spec.Replicas != nil && *cb.Spec.Replicas >= 1 { - v := *cb.Spec.Replicas - cb.Spec.Autoscaling.MinReplicas = &v - } - return nil } diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go index 95343b6e..63a614e4 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go @@ -25,10 +25,8 @@ import ( // TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted is the // end-to-end pin for the defaulter-sweep operator-UX win: applying a -// CacheBackend with the required runtime plus an engine selector and model ID -// must produce a -// fully-defaulted CR with every Phase-1 default stamped — Type=LMCache, -// DeploymentKind=Deployment, Replicas=1, +// typed CacheBackend with type and optional parents omitted must produce a +// fully-defaulted CR with the current defaults stamped — Type=LMCache, // Integration.Role=ReadWrite, Integration.Mode=Offload, // Integration.FailOpen=true, and Observation.FirstEventTimeout=5m. The apiserver in the loop applies // `+kubebuilder:default=` markers; the webhook materialises @@ -120,22 +118,18 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) live := mgr.GetAPIReader() mkNamespace(t, ctx, k8s, "team-a") - // --- Minimum-viable CR: runtime + engineSelector + observation.modelID --- + // --- Minimum typed MP CR with optional/defaulted fields omitted --- // - // An apply with no Type, no DeploymentKind, no Replicas, no Integration - // block, no Storage, no Autoscaling. Every other field must be stamped + // An apply with no Type, Integration, Observation, or remoteStorage. The + // required typed PodLocal shape remains explicit; optional defaults are stamped // by the apiserver (kubebuilder-marker defaults) + the defaulter webhook // (cluster-context defaults). - mvCR := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "minimum", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app.kubernetes.io/name": "vllm"}, - }, - Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "meta-llama/Meta-Llama-3-8B-Instruct"}, - }, - } + mvCR := validPodLocalMPBackend() + mvCR.Name = "minimum" + mvCR.Namespace = "team-a" + mvCR.Spec.Type = "" + mvCR.Spec.Integration = nil + mvCR.Spec.Observation = nil if err := k8s.Create(ctx, mvCR); err != nil { t.Fatalf("minimum-viable CacheBackend should be admitted: %v", err) } @@ -147,7 +141,7 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) t.Fatalf("get back persisted CR: %v", err) } - // --- Phase-1 default surface assertions --- + // --- Current default surface assertions --- // // Each assertion below pins one item from the default sweep. If a future // change drops a default marker or rewrites the defaulter, the @@ -157,12 +151,6 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) if want := cachev1alpha1.CacheBackendTypeLMCache; got.Spec.Type != want { t.Errorf("spec.type = %q, want %q (kubebuilder default)", got.Spec.Type, want) } - if want := cachev1alpha1.CacheBackendDeploymentKindDeployment; got.Spec.DeploymentKind != want { - t.Errorf("spec.deploymentKind = %q, want %q (kubebuilder default)", got.Spec.DeploymentKind, want) - } - if got.Spec.Replicas == nil || *got.Spec.Replicas != 1 { - t.Errorf("spec.replicas = %v, want 1 (kubebuilder default)", got.Spec.Replicas) - } if got.Spec.Integration == nil { t.Fatalf("spec.integration was not materialised by the defaulter; got nil") } @@ -187,13 +175,11 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) // engine-local. The apiserver and webhook must preserve the absence of // deprecated top-level resources rather than claiming a provider workload // this resource did not request. - canonicalCR := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "canonical-host-only", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }, - } + canonicalCR := validPodLocalMPBackend() + canonicalCR.Name = "canonical-host-only" + canonicalCR.Namespace = "team-a" + canonicalCR.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + canonicalCR.Spec.RemoteStorage = nil if err := k8s.Create(ctx, canonicalCR); err != nil { t.Fatalf("canonical host-only CacheBackend should be admitted: %v", err) } @@ -211,18 +197,17 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) // --- Non-clobber pin: an explicit CR overrides every default --- // - // Same minimum-viable shape but with an operator-set replicas and a - // pinned Type. Both values must survive every default layer — proving + // Same shape but with an operator-pinned Type. It must survive every + // default layer, proving // the "defaulter never clobbers" contract holds for the new markers // just as it did for the webhook-stamped defaults before the marker // sweep landed. explicitCR := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "explicit", Namespace: "team-a"}, Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, - HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, - Replicas: i32p(5), + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, + HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ MatchLabels: map[string]string{"app.kubernetes.io/name": "sglang"}, }, @@ -242,15 +227,12 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) if explicit.Spec.Type != cachev1alpha1.CacheBackendTypeSGLangHiCache { t.Errorf("operator type clobbered: got %q, want SGLangHiCache", explicit.Spec.Type) } - if explicit.Spec.Replicas == nil || *explicit.Spec.Replicas != 5 { - t.Errorf("operator replicas clobbered: got %v, want 5", explicit.Spec.Replicas) - } - // --- Final MP API CREATE/UPDATE compatibility --- + // --- Current MP API CREATE/UPDATE compatibility --- // // The real apiserver must accept the new PodLocal shape, preserve it on an - // unrelated update, and reject an update that mixes a legacy flat field into - // the canonical MP contract. + // unrelated update, and reject an update that selects the reserved NodeLocal + // topology before it is implemented. mpCR := validPodLocalMPBackend() mpCR.Name = "podlocal-mp" mpCR.Namespace = "team-a" @@ -272,202 +254,10 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) if err := k8s.Update(ctx, &persistedMP); err != nil { t.Fatalf("unrelated update on PodLocal MP object should be admitted: %v", err) } - persistedMP.Spec.LMCache.WorkerImage = "legacy-worker:test" + persistedMP.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + persistedMP.Spec.LMCache.PodLocal = nil + persistedMP.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} if err := k8s.Update(ctx, &persistedMP); err == nil { - t.Fatal("update mixing legacy workerImage into PodLocal MP should be rejected") - } - - // --- Autoscaling defaulter-computed minReplicas --- - // - // Pins the one non-literal default: when an operator opts into - // autoscaling without pinning the floor, the defaulter computes - // minReplicas from spec.replicas (post-marker-default, so =1 here) - // rather than a hard-coded constant. This is the only field on the - // default that needs cluster context — every other default rides on - // a marker. - hpaCR := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "hpa", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app.kubernetes.io/name": "vllm"}, - }, - Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "meta-llama/Meta-Llama-3-8B-Instruct"}, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, - Autoscaling: &cachev1alpha1.CacheBackendAutoscalingSpec{ - MaxReplicas: 10, - }, - }, - } - if err := k8s.Create(ctx, hpaCR); err != nil { - t.Fatalf("HPA-opted-in CacheBackend should be admitted: %v", err) - } - var hpa cachev1alpha1.CacheBackend - if err := live.Get(ctx, client.ObjectKey{Name: "hpa", Namespace: "team-a"}, &hpa); err != nil { - t.Fatalf("get back hpa CR: %v", err) - } - if hpa.Spec.Autoscaling == nil || hpa.Spec.Autoscaling.MinReplicas == nil || - *hpa.Spec.Autoscaling.MinReplicas != 1 { - t.Errorf("autoscaling.minReplicas = %v, want 1 (= post-default spec.replicas)", - hpa.Spec.Autoscaling.MinReplicas) - } -} - -// TestDefaulter_AutoscalingMinReplicasNotRecomputedOnReplicasUpdate pins the -// FIRST-APPLY-ONLY semantics of the autoscaling.minReplicas default: the -// admission defaulter computes minReplicas from spec.replicas exactly once -// (at the create that opted into autoscaling), and a subsequent update that -// bumps spec.replicas does NOT recompute the floor. This matches the -// standard Kubernetes HPA convention — once an HPA owns the workload, the -// operator-set HPA fields are the source of truth for the autoscaling -// band; spec.replicas edits flow through the HPA controller, not back into -// minReplicas via the admission defaulter. -// -// Without this guarantee an operator who applied at replicas=3 (floor=3 by -// default), then bumped to replicas=5 to manually pre-warm the workload, -// would silently see the autoscaling floor jump to 5 too — turning a -// transient pre-warm into a permanent over-provision. The non-clobber -// contract in the defaulter (refuses to overwrite a non-nil MinReplicas) -// plus the apiserver field manager pinning the previously-stamped value -// together produce the desired "first apply only" behavior; this envtest -// pins it against future regression. -// -// Skips when KUBEBUILDER_ASSETS is unset so default CI stays green; run -// with the same incantation as the test above. -func TestDefaulter_AutoscalingMinReplicasNotRecomputedOnReplicasUpdate(t *testing.T) { - if testing.Short() { - t.Skip("skipping envtest in short mode") - } - if os.Getenv("KUBEBUILDER_ASSETS") == "" { - t.Skip("KUBEBUILDER_ASSETS unset; skipping CacheBackend defaulter envtest") - } - - webhookManifest := filepath.Join("..", "..", "..", "config", "webhook", "manifests.yaml") - - env := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - WebhookInstallOptions: envtest.WebhookInstallOptions{ - Paths: []string{webhookManifest}, - }, - } - cfg, err := env.Start() - if err != nil { - t.Fatalf("envtest.Start: %v", err) - } - t.Cleanup(func() { _ = env.Stop() }) - - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatalf("clientgoscheme.AddToScheme: %v", err) - } - if err := cachev1alpha1.AddToScheme(scheme); err != nil { - t.Fatalf("cachev1alpha1.AddToScheme: %v", err) - } - - wopts := env.WebhookInstallOptions - mgr, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme, - WebhookServer: webhook.NewServer(webhook.Options{ - Host: wopts.LocalServingHost, - Port: wopts.LocalServingPort, - CertDir: wopts.LocalServingCertDir, - }), - Metrics: metricsserver.Options{BindAddress: "0"}, - }) - if err != nil { - t.Fatalf("ctrl.NewManager: %v", err) - } - if err := SetupCacheBackendWebhookWithManager(mgr, defaultShippingRegistry()); err != nil { - t.Fatalf("SetupCacheBackendWebhookWithManager: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - mgrErr := make(chan error, 1) - go func() { mgrErr <- mgr.Start(ctx) }() - t.Cleanup(func() { - cancel() - select { - case err := <-mgrErr: - if err != nil && !isContextCanceledErr(err) { - t.Logf("manager exited with error: %v", err) - } - case <-time.After(5 * time.Second): - t.Logf("manager did not exit within 5s") - } - }) - - if !mgr.GetCache().WaitForCacheSync(ctx) { - t.Fatalf("manager cache did not sync") - } - waitForWebhookPort(t, wopts.LocalServingHost, wopts.LocalServingPort) - - k8s := mgr.GetClient() - live := mgr.GetAPIReader() - mkNamespace(t, ctx, k8s, "team-a") - - // --- Step 1: apply CR with spec.replicas=3, autoscaling.maxReplicas=10, - // no minReplicas. The defaulter computes minReplicas=3 from spec.replicas. - cb := &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "minfloor", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Replicas: i32p(3), - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app.kubernetes.io/name": "vllm"}, - }, - Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "meta-llama/Meta-Llama-3-8B-Instruct"}, - RemoteStorage: &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{}, - }, - Autoscaling: &cachev1alpha1.CacheBackendAutoscalingSpec{ - MaxReplicas: 10, - }, - }, - } - if err := k8s.Create(ctx, cb); err != nil { - t.Fatalf("first-apply CacheBackend should be admitted: %v", err) - } - - // --- Step 2: assert post-apply minReplicas == 3 (first-apply default). - var afterCreate cachev1alpha1.CacheBackend - if err := live.Get(ctx, client.ObjectKey{Name: "minfloor", Namespace: "team-a"}, &afterCreate); err != nil { - t.Fatalf("get back created CR: %v", err) - } - if afterCreate.Spec.Autoscaling == nil || afterCreate.Spec.Autoscaling.MinReplicas == nil || - *afterCreate.Spec.Autoscaling.MinReplicas != 3 { - t.Fatalf("post-apply autoscaling.minReplicas = %v, want 3 (= spec.replicas first-apply default)", - afterCreate.Spec.Autoscaling.MinReplicas) - } - - // --- Step 3: update spec.replicas=5 (operator scales workload manually). - afterCreate.Spec.Replicas = i32p(5) - if err := k8s.Update(ctx, &afterCreate); err != nil { - t.Fatalf("update spec.replicas=5 should be admitted: %v", err) - } - - // --- Step 4: assert post-update minReplicas == 3 (NOT recomputed). - // The non-clobber semantics in the defaulter plus the apiserver field- - // manager ownership of the previously-stamped minReplicas together - // keep the floor anchored at the first-apply value. A regression - // would show minReplicas=5 here (defaulter re-running on every admit - // and re-deriving from spec.replicas). - var afterUpdate cachev1alpha1.CacheBackend - if err := live.Get(ctx, client.ObjectKey{Name: "minfloor", Namespace: "team-a"}, &afterUpdate); err != nil { - t.Fatalf("get back updated CR: %v", err) - } - if afterUpdate.Spec.Replicas == nil || *afterUpdate.Spec.Replicas != 5 { - t.Errorf("spec.replicas update lost: got %v, want 5", afterUpdate.Spec.Replicas) - } - if afterUpdate.Spec.Autoscaling == nil || afterUpdate.Spec.Autoscaling.MinReplicas == nil || - *afterUpdate.Spec.Autoscaling.MinReplicas != 3 { - t.Fatalf("post-update autoscaling.minReplicas = %v, want 3 (operator-owned after first apply; spec.replicas edits must NOT recompute the floor)", - afterUpdate.Spec.Autoscaling.MinReplicas) + t.Fatal("update selecting unimplemented NodeLocal topology should be rejected") } } diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter_test.go b/internal/webhook/v1alpha1/cachebackend_defaulter_test.go index 96731a70..f16e081a 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter_test.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter_test.go @@ -6,166 +6,37 @@ package v1alpha1 import ( "context" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" "time" -) -func TestDefaulter_MaterialisesIntegrationAndObservation(t *testing.T) { - d := &CacheBackendDefaulter{} - cb := newBackend() + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) +func TestDefaulterMaterializesIntegrationAndObservation(t *testing.T) { + cb := &cachev1alpha1.CacheBackend{} + if err := (&CacheBackendDefaulter{}).Default(context.Background(), cb); err != nil { + t.Fatalf("Default: %v", err) } - if cb.Spec.Integration == nil { - t.Fatal("integration block not materialised") + t.Fatal("spec.integration was not materialized") } - if cb.Spec.Observation == nil || cb.Spec.Observation.FirstEventTimeout == nil || cb.Spec.Observation.FirstEventTimeout.Duration != defaultFirstEventTimeout { - t.Fatalf("observation.firstEventTimeout = %v, want %s", cb.Spec.Observation, defaultFirstEventTimeout) + if cb.Spec.Observation == nil || cb.Spec.Observation.FirstEventTimeout == nil || + cb.Spec.Observation.FirstEventTimeout.Duration != 5*time.Minute { + t.Fatalf("observation default = %+v, want 5m", cb.Spec.Observation) } } -func TestDefaulter_PreservesExplicitObservationTimeout(t *testing.T) { - cb := newBackend() - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ - FirstEventTimeout: &metav1.Duration{Duration: 90 * time.Second}, - } - +func TestDefaulterPreservesObservationTimeout(t *testing.T) { + cb := &cachev1alpha1.CacheBackend{Spec: cachev1alpha1.CacheBackendSpec{ + Observation: &cachev1alpha1.CacheBackendObservationSpec{}, + }} + custom := 30 * time.Second + cb.Spec.Observation.FirstEventTimeout = &metav1.Duration{Duration: custom} if err := (&CacheBackendDefaulter{}).Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - if cb.Spec.Observation == nil || cb.Spec.Observation.FirstEventTimeout == nil { - t.Fatalf("observation timeout not materialised: %+v", cb.Spec.Observation) - } - if got := cb.Spec.Observation.FirstEventTimeout.Duration; got != 90*time.Second { - t.Fatalf("observation.firstEventTimeout = %s, want 90s", got) - } -} - -func TestDefaulter_DoesNotClobberOperatorValues(t *testing.T) { - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = i32p(7) - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ - FirstEventTimeout: &metav1.Duration{Duration: 90 * time.Second}, - } - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - // Replicas now defaults via a CRD-schema marker (not the webhook), but the - // non-clobber contract still holds: an operator-set value must survive the - // webhook regardless of which layer applied the default. - if *cb.Spec.Replicas != 7 { - t.Errorf("replicas clobbered: got %d, want 7", *cb.Spec.Replicas) - } - if cb.Spec.Observation.FirstEventTimeout == nil || cb.Spec.Observation.FirstEventTimeout.Duration != 90*time.Second { - t.Errorf("firstEventTimeout clobbered: got %v, want 90s", cb.Spec.Observation.FirstEventTimeout) - } -} - -func TestDefaulter_AutoscalingMinReplicasComputedFromReplicas(t *testing.T) { - // When the operator opts into autoscaling without pinning the floor, the - // defaulter computes minReplicas from spec.replicas so the HPA's lower - // bound follows the baseline declaration rather than a hard-coded - // constant. (spec.replicas itself is stamped by the apiserver from its - // `+kubebuilder:default=1` marker; the unit test seeds it explicitly so - // the assertion does not depend on the marker firing.) - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = i32p(3) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 10} - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - if cb.Spec.Autoscaling.MinReplicas == nil || *cb.Spec.Autoscaling.MinReplicas != 3 { - t.Errorf("autoscaling.minReplicas = %v, want 3 (= spec.replicas)", cb.Spec.Autoscaling.MinReplicas) + t.Fatalf("Default: %v", err) } -} - -func TestDefaulter_AutoscalingMinReplicasNotClobbered(t *testing.T) { - // An operator-set minReplicas survives the defaulter. The non-clobber - // contract extends to every default this handler stamps. - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = i32p(3) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: i32p(2), - MaxReplicas: 10, - } - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - if *cb.Spec.Autoscaling.MinReplicas != 2 { - t.Errorf("autoscaling.minReplicas clobbered: got %d, want 2", *cb.Spec.Autoscaling.MinReplicas) - } -} - -func TestDefaulter_AutoscalingMinReplicasSkippedWhenAutoscalingOff(t *testing.T) { - // No autoscaling = no defaulting. The reconciler's autoscalingFloor - // helper handles the nil-Autoscaling case at runtime; the defaulter - // must not synthesise an autoscaling object operators did not request. - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = i32p(3) - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - if cb.Spec.Autoscaling != nil { - t.Errorf("autoscaling materialised unexpectedly: %+v", cb.Spec.Autoscaling) - } -} - -func TestDefaulter_AutoscalingMinReplicasSkippedWhenReplicasNil(t *testing.T) { - // Defence-in-depth: when a test calls Default() on a raw struct without - // the apiserver in the loop, spec.replicas may still be nil (the schema - // default did not get a chance to fire). The defaulter must leave - // minReplicas alone in that case rather than dereference a nil pointer. - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = nil - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 10} - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - if cb.Spec.Autoscaling.MinReplicas != nil { - t.Errorf("autoscaling.minReplicas should stay nil when spec.replicas is nil; got %v", *cb.Spec.Autoscaling.MinReplicas) - } -} - -func TestDefaulter_AutoscalingMinReplicasSkippedWhenReplicasZero(t *testing.T) { - // spec.replicas permits 0 (scale-to-zero is a valid operator choice), - // but autoscaling.minReplicas carries `+kubebuilder:validation:Minimum=1` - // in the CRD schema. If the defaulter copied a 0 spec.replicas into - // minReplicas the apiserver would then reject the persisted object - // against the schema — a webhook-introduced validation failure on a CR - // the operator did NOT explicitly misconfigure. Refusing to default in - // that case leaves the field unset so the operator's combination of - // `replicas: 0` + opted-in autoscaling surfaces as a missing-required - // field violation against autoscaling itself, which is the actual - // problem. - d := &CacheBackendDefaulter{} - cb := newBackend() - cb.Spec.Replicas = i32p(0) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 10} - - if err := d.Default(context.Background(), cb); err != nil { - t.Fatalf("Default returned error: %v", err) - } - - if cb.Spec.Autoscaling.MinReplicas != nil { - t.Errorf("autoscaling.minReplicas should stay nil when spec.replicas is 0 (would violate schema Minimum=1); got %v", *cb.Spec.Autoscaling.MinReplicas) + if got := cb.Spec.Observation.FirstEventTimeout.Duration; got != custom { + t.Fatalf("timeout = %s, want %s", got, custom) } } diff --git a/internal/webhook/v1alpha1/cachebackend_integration_validation.go b/internal/webhook/v1alpha1/cachebackend_integration_validation.go index 6f1a5bb1..2c39ea88 100644 --- a/internal/webhook/v1alpha1/cachebackend_integration_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_integration_validation.go @@ -13,7 +13,6 @@ import ( adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" "k8s.io/apimachinery/pkg/util/validation/field" "math" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "sort" "strconv" "strings" @@ -103,12 +102,6 @@ func validateSGLangHiCache(cb *cachev1alpha1.CacheBackend) field.ErrorList { "SGLangHiCache must select the engine Pods to inject", )) } - if cb.Spec.Autoscaling != nil { - errs = append(errs, field.Forbidden( - field.NewPath("spec", "autoscaling"), - "SGLangHiCache is engine-local and has no backend workload to autoscale", - )) - } if cachev1alpha1.IntegrationMode(cb.Spec.Integration) != cachev1alpha1.CacheBackendIntegrationModeOffload { errs = append(errs, field.NotSupported( field.NewPath("spec", "integration", "mode"), @@ -205,56 +198,6 @@ func rejectUnsupportedLMCacheRole(cb *cachev1alpha1.CacheBackend) field.ErrorLis } } -// rejectSGLangRedisL2ScaleOut hard-rejects a multi-replica or autoscaled -// (sglang, LMCache) backend. That pair's managed cache-server is a single plain -// Redis L2 store (the SGLang MP worker's --l2-adapter target), and a plain Redis is -// not clustered: a second pod behind the one ClusterIP Service shards the keyspace -// across independent instances, so a key stored via one is a miss via the other and -// the L2 silently partitions. The failure looks like a healthy backend with a poor -// hit rate, so reject at the door rather than warn — the same posture as -// rejectMooncakeMasterScaleOut (a different singleton for a different reason). -// -// Scoped to (sglang, LMCache): vLLM's lm:// server is an ordinary pod-network -// workload that scales, and other pairs are rejected on their own -// (checkRuntimeAdapter). spec.replicas 0 (disabled) and 1 (the singleton) remain -// valid, as is EventsOnly (which provisions no server at all — see the guard). -// The reconciler's clampSingletonReplicas is the backstop for grandfathered -// objects. If SGLang's shared tier gains a clustered store, lift this rule. -func rejectSGLangRedisL2ScaleOut(cb *cachev1alpha1.CacheBackend) field.ErrorList { - storage := cb.Spec.EffectiveRemoteStorage() - if adapterruntime.ResolveRuntimeID(cb) != adapterruntime.RuntimeSGLang || - cb.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache || - storage == nil || - storage.Provider != cachev1alpha1.CacheBackendRemoteStorageProviderRedis || - storage.Ownership != cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged { - return nil - } - // EventsOnly provisions NO cache server at all (the reconciler sheds any owned - // workload and wires only the kvevent-subscriber sidecar), so there is no Redis - // L2 to partition and nothing this rule protects. Rejecting scale-out here would - // be both factually wrong — the message explains a Redis split that cannot happen - // — and gratuitously stricter than the otherwise-identical (vllm, LMCache) - // events-only backend. The rule applies to the Offload path, which is what - // renders the singleton Redis. - if cb.Spec.IsEventsOnly() { - return nil - } - var errs field.ErrorList - if cb.Spec.Replicas != nil && *cb.Spec.Replicas > 1 { - errs = append(errs, field.Invalid( - field.NewPath("spec", "replicas"), *cb.Spec.Replicas, - "the (sglang, LMCache) backend's Redis L2 store is a single non-clustered instance: a second replica behind the one Service shards the keyspace and silently partitions the cache. Set spec.replicas to 0 or 1.", - )) - } - if cb.Spec.Autoscaling != nil { - errs = append(errs, field.Invalid( - field.NewPath("spec", "autoscaling"), cb.Spec.Autoscaling, - "spec.autoscaling is not supported for the (sglang, LMCache) backend: its Redis L2 store is a single non-clustered instance, so scaling it out partitions the cache across independent keyspaces. Remove spec.autoscaling.", - )) - } - return errs -} - // rejectInvalidKernelCheckAnnotation rejects an unrecognized value for the // inferencecache.io/lmcache-kernel-check annotation. The annotation is the // operator's opt-in surface for the engine-side kernel check (auto / @@ -278,69 +221,6 @@ func rejectInvalidKernelCheckAnnotation(cb *cachev1alpha1.CacheBackend) field.Er } } -// mooncakeEngineHostNetworkWarning names the one thing that still stands between a -// Mooncake CacheBackend and working KV transfer. The adapter provisions the master -// correctly (hostNetwork behind a headless Service), but Mooncake's transfer engine -// is a peer-to-peer mesh: the ENGINE pods must run with host networking too. That -// move rewrites a pod the operator owns and hostNetwork is a privilege, so it is -// opt-in via spec.integration.engineHostNetwork rather than injected by default. -// Until the operator opts in, the backend reconciles Ready and moves zero KV — a -// silent failure. Say so out loud at apply time, pointing at the exact field, -// rather than letting them discover it from a flat cache-hit graph. -// -// The text stays within [maxWarningLen]: the API conventions ask for concise -// warnings so clients render them reliably, and a truncated warning is exactly the -// silent failure this exists to prevent. The field and its consequence live here; -// the full rationale (the mesh, node ports, Pod Security ordering) lives in -// docs/design/cachebackend-api.md. -const mooncakeEngineHostNetworkWarning = "Mooncake: set spec.integration.engineHostNetwork=true or engine pods can't join the transfer mesh (Ready, no KV)" - -// maxWarningLen is the concise-warning budget from the Kubernetes API conventions. -// Longer text risks truncation or being dropped by clients. -const maxWarningLen = 120 - -func warnMooncakeEngineHostNetwork(cb *cachev1alpha1.CacheBackend) admission.Warnings { - if !usesMooncakeStorage(cb) { - return nil - } - if enginebinding.EngineHostNetworkRequested(cb) { - // Opted in: the pod webhook moves engine pods onto the host network, so the - // data plane is complete and there is nothing left to warn about. - return nil - } - return admission.Warnings{mooncakeEngineHostNetworkWarning} -} - -// rejectEngineHostNetworkOnBackendThatDoesNotNeedIt keeps -// spec.integration.engineHostNetwork from sitting inert. Only Mooncake's -// peer-to-peer transfer engine needs engine pods on the host network; on any other -// backend the flag silently does nothing while leaving the operator convinced they -// changed the pod's networking. hostNetwork is a privilege — a no-op that *looks* -// like it granted one is worse than a rejection, so reject at the door. -func rejectEngineHostNetworkOnBackendThatDoesNotNeedIt(cb *cachev1alpha1.CacheBackend) field.ErrorList { - if !enginebinding.EngineHostNetworkRequested(cb) || - usesMooncakeStorage(cb) { - return nil - } - provider := "none (host-only)" - if storage := cb.Spec.EffectiveRemoteStorage(); storage != nil { - provider = string(storage.Provider) - } - return field.ErrorList{field.Invalid( - field.NewPath("spec", "integration", "engineHostNetwork"), true, - fmt.Sprintf("spec.integration.engineHostNetwork is only meaningful when the effective remote storage provider is Mooncake, whose transfer engine dials engine pods "+ - "on real node IPs; provider=%s does not need it and the flag would do nothing. Remove it.", provider), - )} -} - -func usesMooncakeStorage(cb *cachev1alpha1.CacheBackend) bool { - if cb == nil { - return false - } - storage := cb.Spec.EffectiveRemoteStorage() - return storage != nil && storage.Provider == cachev1alpha1.CacheBackendRemoteStorageProviderMooncake -} - // checkRuntimeAdapter rejects a CacheBackend whose (runtime, type) pair no // installed runtime adapter supports. The runtime is // resolved through [adapterruntime.ResolveRuntimeID] — the same helper the @@ -388,17 +268,6 @@ func (v *CacheBackendValidator) checkRuntimeAdapter(cb *cachev1alpha1.CacheBacke ), } } - // A declared LMCache topology uses the Phase-1 MP support matrix validated - // by validateLMCacheTopology. The currently shipping runtime adapters still - // describe the legacy data plane (vLLM IP and the SGLang-specific MP spike), - // so consulting SupportsBinding here would incorrectly reject the final - // vLLM+Redis contract before the shared PodLocal renderer lands in Phases - // 2-4. Pod admission remains fail-open until the matching MP adapter is - // implemented; this exception is removed when both adapters expose the final - // binding capabilities. - if cb.Spec.LMCache != nil && cb.Spec.LMCache.Topology != "" { - return nil - } storage := cb.Spec.EffectiveRemoteStorage() protocol, err := backendadapter.ProtocolFor(storage) if err != nil { @@ -460,8 +329,6 @@ func unsupportedPairMessage(engine adapterruntime.RuntimeID, backend cachev1alph // kvevent-subscriber the routing tier needs. // - spec.remoteStorage requests an offload provider that the controller // deliberately removes in events-only mode. -// - spec.autoscaling has no workload to scale — the controller deploys -// nothing for an events-only backend. // // spec.remoteStorage.endpoint is already forbidden for managed ownership by // validateCacheHierarchy, so it needs no events-only-specific check. @@ -483,13 +350,6 @@ func rejectEventsOnlyMisconfiguration(cb *cachev1alpha1.CacheBackend) field.Erro cachev1alpha1.CacheBackendIntegrationModeEventsOnly, cachev1alpha1.CacheBackendTypeLMCache, cb.Spec.Type), )) } - if cb.Spec.Autoscaling != nil { - errs = append(errs, field.Forbidden( - field.NewPath("spec", "autoscaling"), - fmt.Sprintf("events-only backends (spec.integration.mode=%q) provision no server workload, so there is nothing to autoscale", - cachev1alpha1.CacheBackendIntegrationModeEventsOnly), - )) - } if cb.Spec.RemoteStorage != nil { errs = append(errs, field.Forbidden( field.NewPath("spec", "remoteStorage"), @@ -499,73 +359,3 @@ func rejectEventsOnlyMisconfiguration(cb *cachev1alpha1.CacheBackend) field.Erro } return errs } - -// requireExplicitMinReplicasOnScaleToZeroWithAutoscaling rejects the -// combination spec.replicas=0 + spec.autoscaling != nil + -// spec.autoscaling.minReplicas == nil. Without this rule the defaulter -// declines to compute minReplicas (a 0 value would violate the schema's -// Minimum=1), the apiserver accepts the CR with minReplicas left unset, -// and the reconciler's HPA fallback silently picks defaultHPAMinReplicas -// (=1) — so an operator who wrote "scale to zero" gets "scale 1-N" with -// no notification. Forcing the operator to either set the floor -// explicitly or remove the autoscaling block keeps the scale-to-zero -// intent loud at write time. -// -// Bypassed when spec.replicas is nil: the apiserver applies the -// `+kubebuilder:default=1` marker on spec.replicas before this rule -// runs for a CR that came through admission, so a nil here means the -// caller bypassed the apiserver (raw-struct unit-test invocation) and -// the rule has no replicas value to interpret. -func requireExplicitMinReplicasOnScaleToZeroWithAutoscaling(cb *cachev1alpha1.CacheBackend) field.ErrorList { - if cb.Spec.Replicas == nil || *cb.Spec.Replicas != 0 { - return nil - } - if cb.Spec.Autoscaling == nil || cb.Spec.Autoscaling.MinReplicas != nil { - return nil - } - return field.ErrorList{ - field.Required( - field.NewPath("spec", "autoscaling", "minReplicas"), - "spec.replicas=0 with spec.autoscaling enabled requires spec.autoscaling.minReplicas to be set explicitly (must be >=1). "+ - "Set minReplicas to make the autoscaling floor explicit, or remove spec.autoscaling to scale to zero unconditionally.", - ), - } -} - -// rejectMooncakeMasterScaleOut hard-rejects a multi-replica or autoscaled Mooncake -// backend. The Mooncake master is a SINGLETON coordinator that the adapter runs on -// the host network, so a second replica has no good outcome. Co-scheduled, it -// cannot serve: it fails to bind ports the first master already holds on that node -// (in practice the scheduler rejects it earlier still, because the API server -// defaults hostPort=containerPort for hostNetwork pods and the NodePorts predicate -// then trips). Scheduled elsewhere, it comes up as an INDEPENDENT master and -// silently splits the store in two. Both failures land long after admission and -// look like a healthy backend, so reject at the door rather than warn — the same -// posture as the other cross-field invariants here. -// -// spec.replicas 0 (disabled) and 1 (the singleton) remain valid. type=LMCache is -// unaffected: its lm:// server is an ordinary pod-network workload that scales. -func rejectMooncakeMasterScaleOut(cb *cachev1alpha1.CacheBackend) field.ErrorList { - storage := cb.Spec.EffectiveRemoteStorage() - if storage == nil || - storage.Provider != cachev1alpha1.CacheBackendRemoteStorageProviderMooncake || - storage.Ownership != cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged { - return nil - } - var errs field.ErrorList - if cb.Spec.Replicas != nil && *cb.Spec.Replicas > 1 { - errs = append(errs, field.Invalid( - field.NewPath("spec", "replicas"), *cb.Spec.Replicas, - "the Mooncake master is a singleton on the host network: a second replica cannot bind the node ports the first already holds, "+ - "and on a different node it becomes an independent master that silently splits the store. Set spec.replicas to 0 or 1.", - )) - } - if cb.Spec.Autoscaling != nil { - errs = append(errs, field.Invalid( - field.NewPath("spec", "autoscaling"), cb.Spec.Autoscaling, - "spec.autoscaling is not supported for remoteStorage.provider=Mooncake: the master is a singleton on the host network, so scaling it out either cannot bind "+ - "the node's ports or splits the store across independent masters. Remove spec.autoscaling.", - )) - } - return errs -} diff --git a/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go b/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go deleted file mode 100644 index 935a193b..00000000 --- a/internal/webhook/v1alpha1/cachebackend_integration_validation_test.go +++ /dev/null @@ -1,959 +0,0 @@ -// SPDX-FileCopyrightText: 2026 The inference-cache Authors -// -// SPDX-License-Identifier: Apache-2.0 - -package v1alpha1 - -import ( - "context" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - "github.com/cachebox-project/inference-cache/internal/enginebinding" - adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "strings" - "testing" -) - -func TestValidator_SGLangHiCacheAccepted(t *testing.T) { - if _, err := (shippingValidator()).ValidateCreate(context.Background(), newHiCacheBackend()); err != nil { - t.Fatalf("valid SGLangHiCache rejected: %v", err) - } -} - -func TestValidator_CanonicalSGLangHiCacheRejectsRemoteStorage(t *testing.T) { - cb := newHiCacheBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "model-a"} - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, - } - requireInvalidWithCause(t, shippingValidator(), cb, "spec.remoteStorage.provider", - "does not accept remote-storage protocol") -} - -func TestValidator_SGLangHiCacheContract(t *testing.T) { - falseValue := false - size := int32(64) - zero := int32(0) - cases := []struct { - name string - mutate func(*cachev1alpha1.CacheBackend) - want string - }{ - {"missing hiCache", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache = nil }, "spec.hiCache"}, - {"missing capacity", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache.Ratio = "" }, "exactly one"}, - {"both capacities", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache.SizeGB = &size }, "mutually exclusive"}, - {"zero size", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.HiCache.Ratio = "" - cb.Spec.HiCache.SizeGB = &zero - }, "sizeGB"}, - {"invalid ratio", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.HiCache.Ratio = "Inf" }, "ratio"}, - {"wrong runtime", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM }, "spec.runtime"}, - {"missing selector", func(cb *cachev1alpha1.CacheBackend) { cb.Spec.EngineSelector = nil }, "engineSelector.matchLabels"}, - {"events only", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly - }, "integration.mode"}, - {"read only", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Integration.Role = cachev1alpha1.CacheBackendIntegrationRoleReadOnly - }, "integration.role"}, - {"fail closed", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Integration.FailOpen = &falseValue - }, "integration.failOpen"}, - {"autoscaling", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 2} - }, "spec.autoscaling"}, - {"invalid write policy", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.HiCache.WritePolicy = "sometimes" - }, "writePolicy"}, - {"invalid io backend", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.HiCache.IOBackend = "userspace" - }, "ioBackend"}, - {"invalid memory layout", func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.HiCache.MemoryLayout = "tensor_first" - }, "memoryLayout"}, - } - validator := shippingValidator() - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - cb := newHiCacheBackend() - tc.mutate(cb) - _, err := validator.ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), tc.want) { - t.Fatalf("ValidateCreate error = %v, want text %q", err, tc.want) - } - }) - } -} - -func TestValidator_HiCacheBlockRejectedOnOtherTypes(t *testing.T) { - cb := newBackend() - cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"} - _, err := (shippingValidator()).ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), "spec.hiCache") { - t.Fatalf("ValidateCreate error = %v, want hiCache type-scope error", err) - } -} - -func TestValidator_SGLangHiCacheArgsAreReserved(t *testing.T) { - for _, flag := range []string{ - "--enable-hierarchical-cache", - "--hicache-size", - "--hicache-ratio", - "--hicache-write-policy", - "--hicache-io-backend", - "--hicache-mem-layout", - } { - t.Run(flag, func(t *testing.T) { - cb := newHiCacheBackend() - cb.Spec.Integration.EngineOverrides = &cachev1alpha1.EngineInjectionOverrides{ - SuppressArgs: []string{flag}, - } - _, err := (shippingValidator()).ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), flag) || !strings.Contains(err.Error(), "reserved") { - t.Fatalf("ValidateCreate error = %v, want reserved %s", err, flag) - } - }) - } -} - -func TestValidator_MooncakeWarnsUntilEngineHostNetworkOptIn(t *testing.T) { - // Mooncake's transfer engine is a peer-to-peer mesh: engine pods must run with - // hostNetwork or the backend reports Ready and moves zero KV. That move rewrites - // a pod the operator owns, so it is opt-in rather than injected. Until they opt - // in, say so at apply time and name the exact field — otherwise the failure is - // discoverable only from a flat cache-hit graph. - cb := mooncakeBackendWithEngineHostNetwork(false) - t.Run("without opt-in", func(t *testing.T) { - v := shippingValidator() - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("a Mooncake backend must still be admitted (warning, not rejection): %v", err) - } - if len(warnings) != 1 || !strings.Contains(warnings[0], "spec.integration.engineHostNetwork=true") { - t.Fatalf("create warnings = %v, want one warning naming the opt-in field", warnings) - } - - // It must persist across updates, not only on first apply — an operator who - // edits the CR later should still be told. - warnings, err = v.ValidateUpdate(context.Background(), cb, cb) - if err != nil { - t.Fatalf("a Mooncake update must still be admitted: %v", err) - } - if len(warnings) != 1 { - t.Fatalf("update warnings = %v, want the engine-hostNetwork warning", warnings) - } - }) -} - -func TestValidator_MooncakeOptInSilencesTheWarning(t *testing.T) { - // Once the operator opts in, the pod webhook completes the data plane. A warning - // that keeps firing after the gap is closed trains operators to ignore warnings. - cb := mooncakeBackendWithEngineHostNetwork(true) - t.Run("with opt-in", func(t *testing.T) { - v := shippingValidator() - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("an opted-in Mooncake backend must be admitted: %v", err) - } - if len(warnings) != 0 { - t.Fatalf("warnings = %v, want none once engineHostNetwork is set", warnings) - } - }) -} - -func TestValidator_EngineHostNetworkRejectedOnBackendThatDoesNotNeedIt(t *testing.T) { - // The flag would silently do nothing on a pod-network backend while leaving the - // operator convinced they had granted their engine host networking. hostNetwork - // is a privilege — a no-op that looks like it granted one is worse than an error. - v := shippingValidator() - cb := mooncakeBackendWithEngineHostNetwork(true) - cb.Spec.RemoteStorage = nil - requireInvalidWithCause(t, v, cb, "spec.integration.engineHostNetwork", - "only meaningful when the effective remote storage provider is Mooncake") -} - -func TestValidator_EngineHostNetworkGoesInertWhenTypeFlipsAwayFromMooncake(t *testing.T) { - // The realistic way the flag rots: a Mooncake backend legitimately carrying - // engineHostNetwork=true is retyped to LMCache, and the flag rides along as a - // no-op that reads like a granted privilege. - // - // Worth pinning explicitly because ValidateUpdate only rejects errors the new - // object *introduces* — errors already present on the old object are filtered - // out. The old object here (Mooncake + flag) is valid, so the error IS newly - // introduced and must be caught. Nothing about that is obvious from the rule. - v := shippingValidator() - old := mooncakeBackendWithEngineHostNetwork(true) - newCB := mooncakeBackendWithEngineHostNetwork(true) - newCB.Spec.RemoteStorage = nil - requireUpdateInvalidWithCause(t, v, old, newCB, "spec.integration.engineHostNetwork", - "only meaningful when the effective remote storage provider is Mooncake") -} - -func TestValidator_DroppingEngineHostNetworkWithTheTypeFlipIsAccepted(t *testing.T) { - // The escape hatch the rejection above implies: retyping away from Mooncake is - // fine as long as the flag goes with it. If this failed, the rule would have - // wedged the object — rejecting both keeping and dropping the flag. - v := shippingValidator() - old := mooncakeBackendWithEngineHostNetwork(true) - newCB := mooncakeBackendWithEngineHostNetwork(false) - newCB.Spec.RemoteStorage = nil - if _, err := v.ValidateUpdate(context.Background(), old, newCB); err != nil { - t.Fatalf("retyping away from Mooncake while dropping engineHostNetwork must be accepted, got: %v", err) - } -} - -func TestValidator_EngineHostNetworkCannotGoInertViaEventsOnly(t *testing.T) { - // The other way engineHostNetwork could end up inert: an events-only backend - // wires no KV connector, so the Pod webhook never calls InjectEngineConfig and - // the flag would do nothing. Today that combination is already unreachable — - // rejectEventsOnlyMisconfiguration forbids EventsOnly on any managed type but - // LMCache, and engineHostNetwork is rejected on any type but Mooncake, so the - // two rules cross. - // - // Pinned because that safety is emergent, not stated: it comes from two - // independent rules meeting. Loosening either one — allowing events-only - // Mooncake, say — would silently open the inert-flag hole this asserts shut. - v := shippingValidator() - cb := mooncakeBackendWithEngineHostNetwork(true) - cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly - requireInvalidWithCause(t, v, cb, "spec.remoteStorage", - "provision no remote-storage provider") -} - -func TestValidator_NonMooncakeEmitsNoHostNetworkWarning(t *testing.T) { - // Blast radius: the DEFAULT (vLLM) LMCache pairing — engine unset defaults to - // vLLM — must stay warning-free (the Mooncake mesh warning does not apply to it). - v := shippingValidator() - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("LMCache backend must be admitted: %v", err) - } - if len(warnings) != 0 { - t.Fatalf("LMCache warnings = %v, want none", warnings) - } -} - -func TestValidator_SGLangLMCacheEmitsNoWarning(t *testing.T) { - // The (sglang, LMCache) adapter now renders the working LMCache MP-mode data - // plane (node-local MP-worker sidecar + config-file wire → managed Redis L2), so - // the old "misconfigured lm:// wiring" advisory is gone — the pair must be - // warning-free on both create and update. Assert the FULL list is empty so a - // resurrected or accidental new warning is caught. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("(sglang, LMCache) must be admitted: %v", err) - } - if len(warnings) != 0 { - t.Fatalf("(sglang, LMCache) must be warning-free, got: %v", warnings) - } - - warnings, err = v.ValidateUpdate(context.Background(), cb, cb) - if err != nil { - t.Fatalf("(sglang, LMCache) update must be admitted: %v", err) - } - if len(warnings) != 0 { - t.Fatalf("(sglang, LMCache) update must be warning-free, got: %v", warnings) - } -} - -func TestValidator_VLLMLMCacheEmitsNoSGLangWarning(t *testing.T) { - // Blast radius: the sglang MP-mode warning must not fire on (vllm, LMCache), - // which drives the lm:// server correctly. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("(vllm, LMCache) must be admitted: %v", err) - } - // The contract is warning-FREE, not merely "no MP-mode warning" — assert the - // full list is empty so a reworded warning or an accidental new one is caught. - if len(warnings) != 0 { - t.Fatalf("(vllm, LMCache) must be warning-free, got: %v", warnings) - } -} - -func TestValidator_SGLangEventsOnlyEmitsNoDataPlaneWarning(t *testing.T) { - // EventsOnly (tier-1 routing) provisions no server and injects no LMCache - // connector — only the observation sidecar — so the lm://-vs-MP mismatch is - // absent and the warning must not fire. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - } - warnings, err := v.ValidateCreate(context.Background(), cb) - if err != nil { - t.Fatalf("(sglang, LMCache, EventsOnly) must be admitted: %v", err) - } - for _, w := range warnings { - if strings.Contains(w, "MP mode") { - t.Fatalf("events-only backend got the sglang MP-mode warning: %q", w) - } - } -} - -func TestValidator_MooncakeMultiReplicaRejected(t *testing.T) { - // The Mooncake master is a singleton on the host network: a second replica cannot - // bind the node ports the first already holds, and on a different node it comes up - // as an independent master and silently splits the store. Both failures surface - // long after the object looks healthy, so admission rejects them at write time. - v := shippingValidator() - cb := newBackend() - setCanonicalMooncakeStorage(cb) - two := int32(2) - cb.Spec.Replicas = &two - requireInvalidWithCause(t, v, cb, "spec.replicas", "singleton on the host network") -} - -func TestValidator_MooncakeAutoscalingRejected(t *testing.T) { - v := shippingValidator() - cb := newBackend() - setCanonicalMooncakeStorage(cb) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{} - requireInvalidWithCause(t, v, cb, "spec.autoscaling", "not supported for remoteStorage.provider=Mooncake") -} - -func TestValidator_MooncakeSingletonAndDisabledReplicasAccepted(t *testing.T) { - // 1 is the singleton; 0 is the "disabled" case. Neither can split the store. - v := shippingValidator() - for _, replicas := range []int32{0, 1} { - cb := newBackend() - setCanonicalMooncakeStorage(cb) - r := replicas - cb.Spec.Replicas = &r - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("Mooncake with spec.replicas=%d must be admitted: %v", replicas, err) - } - } -} - -func TestValidator_LMCacheScaleOutUnaffectedByMooncakeRule(t *testing.T) { - // Blast radius: the lm:// server is an ordinary pod-network workload and must - // keep scaling out (and autoscaling) normally. - v := shippingValidator() - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - three := int32(3) - cb.Spec.Replicas = &three - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{} - managedLMCacheServer(cb) - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("multi-replica autoscaled LMCache must be admitted: %v", err) - } -} - -func TestValidator_SGLangRedisL2MultiReplicaRejected(t *testing.T) { - // The (sglang, LMCache) backend's cache-server is a single non-clustered Redis L2 - // (the MP worker's --l2-adapter target). A second pod behind the one Service - // shards the keyspace across independent instances, so a key stored via one is a - // miss via the other — the L2 silently partitions. Reject at write time. - v := shippingValidator() - cb := sglangLMCacheBackend() - two := int32(2) - cb.Spec.Replicas = &two - requireInvalidWithCause(t, v, cb, "spec.replicas", "single non-clustered instance") -} - -func TestValidator_SGLangRedisL2AutoscalingRejected(t *testing.T) { - v := shippingValidator() - cb := sglangLMCacheBackend() - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{} - requireInvalidWithCause(t, v, cb, "spec.autoscaling", "not supported for the (sglang, LMCache) backend") -} - -func TestValidator_SGLangRedisL2SingletonAndDisabledAccepted(t *testing.T) { - // 1 is the singleton; 0 is "disabled". Neither partitions the keyspace. - v := shippingValidator() - for _, replicas := range []int32{0, 1} { - cb := sglangLMCacheBackend() - r := replicas - cb.Spec.Replicas = &r - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("(sglang, LMCache) with spec.replicas=%d must be admitted: %v", replicas, err) - } - } -} - -func TestValidator_SGLangEventsOnlyScaleOutAccepted(t *testing.T) { - // EventsOnly provisions NO cache server (the reconciler sheds any owned - // workload), so there is no Redis L2 to partition — the singleton rule must not - // fire. Rejecting here would be factually wrong (the message explains a Redis - // split that cannot happen) and would make SGLang gratuitously stricter than an - // otherwise-identical (vllm, LMCache) events-only backend. - v := shippingValidator() - - t.Run("multi-replica is admitted", func(t *testing.T) { - cb := sglangLMCacheBackend() - cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly - cb.Spec.RemoteStorage = nil - cb.Spec.Replicas = i32p(3) - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("(sglang, LMCache) EventsOnly with replicas=3 must be admitted (no Redis is provisioned): %v", err) - } - }) - - t.Run("autoscaling is rejected by the ENGINE-AGNOSTIC events-only rule, not the Redis one", func(t *testing.T) { - // Autoscaling on EventsOnly is rejected either way — but it must be for the - // generic "no server workload to autoscale" reason that applies to every - // engine, NOT the SGLang Redis-partitioning reason (which would be wrong here - // and would make SGLang stricter than vLLM). Pinning the reason is the point. - cb := sglangLMCacheBackend() - cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly - cb.Spec.RemoteStorage = nil - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("EventsOnly + autoscaling should be rejected (nothing to autoscale)") - } - if !strings.Contains(err.Error(), "provision no server workload") { - t.Fatalf("want the engine-agnostic events-only reason, got: %v", err) - } - if strings.Contains(err.Error(), "non-clustered") { - t.Fatalf("the SGLang Redis-partitioning rule fired on an events-only backend, which provisions no Redis: %v", err) - } - }) -} - -func TestValidator_VLLMLMCacheScaleOutUnaffectedBySGLangRule(t *testing.T) { - // Blast radius: vLLM's lm:// server is an ordinary pod-network workload and must - // keep scaling out (and autoscaling) — the singleton rule is (sglang, LMCache)-only. - v := shippingValidator() - cb := newBackend() // Type=LMCache, engine defaults to vllm - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - three := int32(3) - cb.Spec.Replicas = &three - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{} - managedLMCacheServer(cb) - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("multi-replica autoscaled (vllm, LMCache) must be admitted: %v", err) - } -} - -func TestValidator_InvalidKernelCheckAnnotationRejected(t *testing.T) { - v := shippingValidator() - - // A typo for "strict" would silently fall back to "auto" (report-only) and - // disable the fail-closed gate — reject it at admission instead. - bad := newBackend() - bad.Annotations = map[string]string{enginebinding.AnnotationLMCacheKernelCheck: "strcit"} - requireInvalidWithCause(t, v, bad, "metadata.annotations[inferencecache.io/lmcache-kernel-check]", "must be one of") - - // Every known value — and an unset annotation — is accepted. - for _, val := range []string{ - enginebinding.KernelCheckModeAuto, - enginebinding.KernelCheckModeReportOnly, - enginebinding.KernelCheckModeStrict, - enginebinding.KernelCheckModeOff, - "", // explicit empty == unset - } { - ok := newBackend() - ok.Annotations = map[string]string{enginebinding.AnnotationLMCacheKernelCheck: val} - if _, err := v.ValidateCreate(context.Background(), ok); err != nil { - t.Fatalf("valid kernel-check annotation %q rejected: %v", val, err) - } - } - if _, err := v.ValidateCreate(context.Background(), newBackend()); err != nil { - t.Fatalf("unset kernel-check annotation rejected: %v", err) - } -} - -func TestValidator_ReplicasZeroWithAutoscalingAndNilMinReplicasRejected(t *testing.T) { - // spec.replicas=0 + spec.autoscaling enabled + nil minReplicas is the - // silent-HPA-fallback-to-1 trap: the defaulter declines to default - // minReplicas (a 0 value would violate the schema's Minimum=1), the - // apiserver accepts the CR with minReplicas unset, and the reconciler's - // HPA fallback picks defaultHPAMinReplicas=1 — overriding the operator's - // "scale to zero" intent without notification. Admission must reject - // the combination so the operator either sets the floor explicitly or - // removes the autoscaling block to truly scale to zero. - v := shippingValidator() - cb := newBackend() - cb.Spec.Replicas = i32p(0) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 10} - requireInvalidWithCause(t, v, cb, "spec.autoscaling.minReplicas", - "spec.replicas=0 with spec.autoscaling enabled requires spec.autoscaling.minReplicas") -} - -func TestValidator_ReplicasZeroWithAutoscalingAndExplicitMinReplicasAdmitted(t *testing.T) { - // Operator who pairs replicas=0 with autoscaling sets minReplicas - // explicitly to declare the intended HPA floor. With minReplicas=1 the - // HPA scales the workload back up to 1 immediately (minReplicas=1 means - // "never below one"); the test pins that the admission rule fires only - // on the nil-minReplicas trap, not on the explicit-floor case. (CRD - // schema enforces Minimum=1 on minReplicas, so the smallest legal - // explicit value here is 1; true scale-to-zero requires removing the - // autoscaling block entirely, which the next test covers.) - v := shippingValidator() - cb := newBackend() - cb.Spec.Replicas = i32p(0) - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MinReplicas: i32p(1), - MaxReplicas: 10, - } - managedLMCacheServer(cb) - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("replicas=0 + autoscaling + explicit minReplicas rejected: %v", err) - } -} - -func TestValidator_ReplicasZeroWithoutAutoscalingAdmitted(t *testing.T) { - // Pure scale-to-zero (no autoscaling block) is allowed. The HPA-fallback - // trap only applies when autoscaling is opted into. - v := shippingValidator() - cb := newBackend() - cb.Spec.Replicas = i32p(0) - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("replicas=0 without autoscaling rejected: %v", err) - } -} - -func TestValidator_RuntimeAdapter_VLLMPlusLMCacheAdmitted(t *testing.T) { - // Happy path: an explicit (vLLM, LMCache) pair the stub registry - // supports must be admitted. Pins the C7 check's positive side so a - // regression doesn't silently start rejecting it. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("vLLM+LMCache rejected: %v", err) - } -} - -func TestDefaultShippingRegistryResolvesSGLangLMCache(t *testing.T) { - // Registration check: the real shipping registry (the set the running - // controller installs) must resolve the (sglang, LMCache) pair to an - // adapter, and surface sglang/LMCache in its SupportedPairs so admission - // error messages list it as a candidate. Exercises the real - // defaultShippingRegistry rather than a stub so a regression that drops the - // SGLang registration from any of the three wiring sites is caught here. - r := defaultShippingRegistry() - cb := newBackend() // type=LMCache - if _, err := r.Select(adapterruntime.RuntimeSGLang, cb); err != nil { - t.Fatalf("shipping registry does not resolve (sglang, LMCache): %v", err) - } - found := false - for _, p := range r.SupportedPairs() { - if p.Runtime == adapterruntime.RuntimeSGLang && p.Backend == cachev1alpha1.CacheBackendTypeLMCache { - found = true - break - } - } - if !found { - t.Fatalf("SupportedPairs missing sglang/LMCache; got %v", r.SupportedPairs()) - } -} - -func TestValidator_RuntimeAdapter_SGLangPlusLMCacheAdmitted(t *testing.T) { - // DoD: admission accepts (sglang, LMCache) once the adapter is registered. - // Runs through the real shipping registry so the test fails if the SGLang - // adapter is not actually wired into the validator's registry. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("sglang+LMCache rejected: %v", err) - } -} - -func TestValidator_RuntimeAdapter_SGLangPlusExternalRejected(t *testing.T) { - // The SGLang adapter supports only (sglang, LMCache) — a (sglang, External) - // pair must still be rejected (no adapter claims it), and the message must - // list sglang/LMCache among the supported candidates so the operator sees - // the actionable alternative. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("expected (sglang, External) to be rejected") - } - statusErr, ok := err.(*apierrors.StatusError) - if !ok { - t.Fatalf("expected *apierrors.StatusError, got %T: %v", err, err) - } - var match *metav1.StatusCause - for i := range statusErr.Status().Details.Causes { - if statusErr.Status().Details.Causes[i].Field == "spec.runtime" { - match = &statusErr.Status().Details.Causes[i] - break - } - } - if match == nil { - t.Fatalf("no cause on spec.runtime; got: %+v", statusErr.Status().Details.Causes) - } - for _, want := range []string{"sglang", "unsupported", "sglang/LMCache"} { - if !strings.Contains(match.Message, want) { - t.Errorf("rejection message missing %q; got %q", want, match.Message) - } - } -} - -func TestValidator_RuntimeAdapter_VLLMPlusUnsupportedTypeRejected(t *testing.T) { - // Rejection path: a (vLLM, ) pair no installed adapter - // supports must be rejected with a message that names BOTH sides of - // the offending pair and lists the supported pairs so the user has - // an actionable next step. The arbitrary value below is unsupported by any - // shipping adapter. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("expected vLLM+unsupported to be rejected") - } - statusErr, ok := err.(*apierrors.StatusError) - if !ok { - t.Fatalf("expected *apierrors.StatusError, got %T: %v", err, err) - } - if statusErr.Status().Details == nil || len(statusErr.Status().Details.Causes) == 0 { - t.Fatalf("Invalid status carried no causes: %v", statusErr.Status()) - } - var match *metav1.StatusCause - causes := statusErr.Status().Details.Causes - for i := range causes { - if causes[i].Field == "spec.runtime" { - match = &causes[i] - break - } - } - if match == nil { - t.Fatalf("no cause on spec.runtime; got: %+v", causes) - } - for _, want := range []string{"vllm", "unsupported", "vllm/LMCache"} { - if !strings.Contains(match.Message, want) { - t.Errorf("rejection message missing %q; got %q", want, match.Message) - } - } -} - -func TestValidator_RuntimeAdapter_UnknownEngineRejected(t *testing.T) { - // An engine name no adapter handles must also be rejected — guards - // against a typo (`engin: vllmm`) silently riding through admission - // and only failing at reconcile. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntime("vllmm") - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.runtime", - "engine=\"vllmm\"") -} - -func TestValidator_RuntimeAdapter_EngineNormalisedToLowerCase(t *testing.T) { - // The reconciler downcases the engine string before looking up an - // adapter; admission must do the same so a CR that spells "VLLM" is - // not admitted by one layer and rejected by the other. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("VLLM (uppercase) + LMCache rejected: %v", err) - } -} - -func TestValidator_RuntimeAdapter_EmptyEngineDefaultsToVLLM(t *testing.T) { - // Engine is optional on the CRD; the reconciler and pod webhook - // default it to vLLM via adapterruntime.ResolveRuntimeID, so - // admission must use the same defaulting or pairs like - // an unsupported type with no engine slip past the webhook and only - // fail at reconcile (the exact gap C7 closes). The value has no adapter - // in any registry, so it stays a genuinely-unsupported example. - // - // With LMCache the default vLLM pair is supported → admit. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache, no Integration block - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("LMCache + defaulted vLLM engine rejected: %v", err) - } -} - -func TestValidator_RuntimeAdapter_EmptyEngineWithUnsupportedTypeRejected(t *testing.T) { - // Counterpart to the previous test: the default vLLM resolution - // must also fire C7 — an unsupported type with no engine must be - // rejected at admission, since the reconciler would otherwise try - // vllm/unsupported and fall back to unmanaged. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") - requireInvalidWithCause(t, v, cb, "spec.runtime", - "backend=\"unsupported\"") -} - -func TestValidator_RuntimeAdapter_EmptyTypeSkipsCheck(t *testing.T) { - // Mirror edge case: an empty type must not trigger C7 either, for - // the same "defer to required-field validation" reason. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - cb.Spec.Type = "" - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("empty type must not trigger C7; got %v", err) - } -} - -func TestValidator_RuntimeAdapter_ExternalWithSupportedEngineAdmitted(t *testing.T) { - // External ownership does not change runtime-adapter selection: this remains - // a supported vLLM/LMCache pair with an external remote binding. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - setCanonicalExternalStorage(cb, "team-a-cache.team-a.svc.cluster.local:9000") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("External with engine=vllm rejected by C7: %v", err) - } -} - -func TestValidator_RuntimeAdapter_ExternalWithUnsupportedEngineRejected(t *testing.T) { - // External + sglang is admittable on shape (endpoint present, type set) - // but no adapter in the registry handles that pair, so the pod webhook - // would fail-open and never inject — the engine boots un-wired to the - // external cache. Reject at admission with a useful error instead of - // letting the silent miss happen. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - setCanonicalExternalStorage(cb, "team-a-cache.team-a.svc.cluster.local:9000") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.runtime", - "backend=\"LMCache\"") -} - -func TestValidator_RuntimeAdapter_UpdateAlsoChecks(t *testing.T) { - // ValidateUpdate runs the same check as ValidateCreate — a kubectl - // edit that flips engine to something the registry doesn't support - // must be rejected just as it would on create. - v := &CacheBackendValidator{Registry: stubRegistry()} - old := newBackend() - old.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - newCB := old.DeepCopy() - newCB.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") - - _, err := v.ValidateUpdate(context.Background(), old, newCB) - if err == nil || !apierrors.IsInvalid(err) { - t.Fatalf("expected Invalid on update with unsupported pair, got %v", err) - } -} - -func TestValidator_RuntimeAdapter_DeleteSkipsCheck(t *testing.T) { - // Deletion of a CR that would now be rejected (e.g. registry shrank - // since admission) must still be allowed so operators can clean up. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - cb.Spec.Type = cachev1alpha1.CacheBackendType("unsupported") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateDelete(context.Background(), cb); err != nil { - t.Fatalf("ValidateDelete rejected unsupported pair: %v", err) - } -} - -func TestValidator_RuntimeAdapter_ShippingRegistryAdmitsExternal(t *testing.T) { - // The explicitly injected shipping registry must admit the same pair the - // running controller can reconcile and inject. - v := shippingValidator() - cb := newBackend() - setCanonicalExternalStorage(cb, "ext.example.com:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("shipping registry rejected vLLM+External: %v", err) - } -} - -func TestValidator_RuntimeAdapter_NilRegistryRejectsMisconfiguration(t *testing.T) { - v := &CacheBackendValidator{} - _, err := v.ValidateCreate(context.Background(), newBackend()) - if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "registry is not configured") { - t.Fatalf("ValidateCreate error = %v, want invalid missing-registry error", err) - } -} - -func TestValidator_RuntimeAdapter_NilRegistryFallsBackToDefault(t *testing.T) { - // A zero-value validator (Registry nil) must still run the C7 check - // against the complete built-in registry — the production safety net for - // cmd/controller wiring drift. The built-in registry ships the vLLM+LMCache - // adapter, so the happy pair admits. - v := shippingValidator() - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("nil-registry fallback rejected vLLM+LMCache: %v", err) - } -} - -func TestValidator_RuntimeAdapter_VLLMPlusMooncakeAdmittedViaShippingRegistry(t *testing.T) { - // Mooncake is a remote binding for the vLLM/LMCache runtime pair, so the - // shipping registry must admit it without a provider-specific runtime adapter. - v := shippingValidator() - cb := newBackend() - setCanonicalMooncakeStorage(cb) - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("shipping registry rejected vLLM/LMCache with Mooncake binding: %v", err) - } -} - -func TestValidator_LMCacheDirectionalRolesRejected(t *testing.T) { - // Neither shipping LMCache integration can currently honor directional - // roles: SGLang has no split, while the validated vLLM MP connector ignores - // kv_consumer/kv_producer behaviorally. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - for _, runtime := range []cachev1alpha1.CacheBackendRuntime{ - cachev1alpha1.CacheBackendRuntimeVLLM, - cachev1alpha1.CacheBackendRuntimeSGLang, - } { - for _, role := range []cachev1alpha1.CacheBackendIntegrationRole{ - cachev1alpha1.CacheBackendIntegrationRoleReadOnly, - cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, - } { - t.Run(string(runtime)+"/"+string(role), func(t *testing.T) { - cb := newBackend() // type=LMCache - cb.Spec.Runtime = runtime - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{Role: role} - requireInvalidWithCause(t, v, cb, "spec.integration.role", "directional cache access") - }) - } - } -} - -func TestValidator_LMCacheReadWriteAndUnsetAdmitted(t *testing.T) { - // ReadWrite (and unset, which defaults to ReadWrite) are the only currently - // honored LMCache roles for both engines. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - for _, runtime := range []cachev1alpha1.CacheBackendRuntime{ - cachev1alpha1.CacheBackendRuntimeVLLM, - cachev1alpha1.CacheBackendRuntimeSGLang, - } { - for _, integ := range []*cachev1alpha1.CacheBackendIntegrationSpec{ - {}, // role unset - {Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, - } { - cb := newBackend() - cb.Spec.Runtime = runtime - cb.Spec.Integration = integ - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("runtime=%q role=%q rejected: %v", runtime, integ.Role, err) - } - } - } -} - -func TestValidator_EventsOnly_ExternalTypeRejected(t *testing.T) { - // events-only wires no KV connector, while External provisions an - // operator-run offload server a connector would dial — the two are - // contradictory. The rejection must point at spec.integration.mode (the - // knob the operator flipped), not at spec.type. Use the registry that - // uses the shipping LMCache adapter; the events-only remote-storage rule is - // the one that must fire. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - setCanonicalExternalStorage(cb, "team-a-cache.team-a.svc.cluster.local:9000") - cb.Spec.Integration = eventsOnlyIntegration() - requireInvalidWithCause(t, v, cb, "spec.remoteStorage", - "provision no remote-storage provider") -} - -func TestValidator_EventsOnly_AutoscalingRejected(t *testing.T) { - // An events-only backend provisions no server workload, so an autoscaling - // spec has nothing to scale; admission rejects it against spec.autoscaling. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Integration = eventsOnlyIntegration() - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{ - MaxReplicas: 3, - } - requireInvalidWithCause(t, v, cb, "spec.autoscaling", - "nothing to autoscale") -} - -func TestValidator_EventsOnly_RemoteStorageRejected(t *testing.T) { - v := shippingValidator() - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = eventsOnlyIntegration() - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{ - Image: "cache-server:test", - }, - } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage", - "provision no remote-storage provider") -} - -func TestValidator_EventsOnly_LMCacheAdmitted(t *testing.T) { - // The supported events-only shape: type=LMCache (whose adapter supplies - // the kvevent-subscriber the routing tier needs), no autoscaling, no - // endpoint. The events-only rule must accept it cleanly — events-only is - // the lighter routing-only deployment, not a misconfiguration. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Integration = eventsOnlyIntegration() - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("events-only LMCache (no autoscaling, no endpoint) rejected: %v", err) - } -} - -func TestValidator_EventsOnly_MooncakeRejected(t *testing.T) { - // EventsOnly is LMCache-only. A managed Mooncake backend in events-only mode - // would stand up a mooncake_master store but wire no KV connector to use it - // — a contradiction. This must be rejected EXPLICITLY now that the - // (vLLM, Mooncake) adapter is registered: the runtime-adapter check ADMITS - // the pair (Mooncake is supported), so without the events-only rule's - // type check the CR would slip through and reconcile as active events-only. - // Use the explicitly injected built-in shipping registry so the - // runtime-adapter check passes and - // the events-only rule is the one that fires, on spec.integration.mode. - v := shippingValidator() - cb := newBackend() - setCanonicalMooncakeStorage(cb) - cb.Spec.Integration = eventsOnlyIntegration() - requireInvalidWithCause(t, v, cb, "spec.remoteStorage", - "provision no remote-storage provider") -} - -func TestValidator_EventsOnly_OffloadDefaultLMCacheAdmitted(t *testing.T) { - // Regression: the default integration mode (Offload) on an LMCache backend - // — set explicitly here, but it is also the +kubebuilder default — must be - // untouched by the events-only rule. Guards against the rule firing on the - // Offload path. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() // type=LMCache - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, - } - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("Offload (default) LMCache rejected: %v", err) - } -} diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index d42ad589..4d65e667 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -24,67 +24,26 @@ const ( var sha256ImagePattern = regexp.MustCompile(`^[^[:space:]@]+@sha256:[a-f0-9]{64}$`) -// validateLMCacheTopology enforces the canonical MP-only LMCache shape while -// leaving a topology-less legacy object untouched during repository migration. -// The presence of topology/podLocal/nodeLocal is the explicit boundary between -// the old flat inputs and the new API; the two shapes can never be mixed. +// validateLMCacheTopology enforces the canonical MP-only LMCache shape. func validateLMCacheTopology(cb *cachev1alpha1.CacheBackend) field.ErrorList { - if cb == nil || cb.Spec.LMCache == nil { + if cb == nil || cb.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { return nil } - - lm := cb.Spec.LMCache lmPath := field.NewPath("spec", "lmCache") - hasMPShape := lm.Topology != "" || lm.PodLocal != nil || lm.NodeLocal != nil - if !hasMPShape { - return nil // grandfathered flat-field/IP shape - } - - var errs field.ErrorList - if cb.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { - // validateCacheHierarchy owns the clearer type error. + if cb.Spec.IsEventsOnly() { + if cb.Spec.LMCache != nil { + return field.ErrorList{field.Forbidden(lmPath, + "LMCache topology is invalid with integration.mode=EventsOnly because that mode injects no KV connector or MP server")} + } return nil } - if cb.Spec.IsEventsOnly() { - errs = append(errs, field.Forbidden(lmPath, - "LMCache topology is invalid with integration.mode=EventsOnly because that mode injects no KV connector or MP server")) + if cb.Spec.LMCache == nil { + return field.ErrorList{field.Required(lmPath, "required for the LMCache multiprocess data plane")} } - // Flat fields are compatibility inputs, not alternate spellings for MP. - if lm.HostMemory != nil { - errs = append(errs, field.Forbidden(lmPath.Child("hostMemory"), - "legacy flat field cannot be mixed with the MP topology; use podLocal.server.l1Capacity")) - } - if strings.TrimSpace(lm.WorkerImage) != "" { - errs = append(errs, field.Forbidden(lmPath.Child("workerImage"), - "legacy flat field cannot be mixed with the MP topology; use podLocal.server.image")) - } - if lm.WorkerPort != nil { - errs = append(errs, field.Forbidden(lmPath.Child("workerPort"), - "legacy flat field cannot be mixed with the MP topology; use podLocal.server.port")) - } - if strings.TrimSpace(lm.RemoteSerde) != "" { - errs = append(errs, field.Forbidden(lmPath.Child("remoteSerde"), - "remoteSerde belongs to the legacy in-process connector and is not supported by LMCache MP")) - } + lm := cb.Spec.LMCache - storage := cb.Spec.EffectiveRemoteStorage() - if storage != nil { - switch storage.Provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: - // Redis/RESP is the initial shared L3 for both engines. - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - errs = append(errs, field.NotSupported( - field.NewPath("spec", "remoteStorage", "provider"), storage.Provider, - []string{string(cachev1alpha1.CacheBackendRemoteStorageProviderRedis)}, - )) - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - errs = append(errs, field.NotSupported( - field.NewPath("spec", "remoteStorage", "provider"), storage.Provider, - []string{string(cachev1alpha1.CacheBackendRemoteStorageProviderRedis)}, - )) - } - } + var errs field.ErrorList switch lm.Topology { case "": diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index eb2c94c1..42863281 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -9,285 +9,38 @@ import ( "strings" "testing" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" -) + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -const testMPServerImage = "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) func validPodLocalMPBackend() *cachev1alpha1.CacheBackend { - l1 := resource.MustParse("1Gi") return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "team-a"}, Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, LMCache: &cachev1alpha1.LMCacheEngineSpec{ Topology: cachev1alpha1.LMCacheTopologyPodLocal, - PodLocal: &cachev1alpha1.LMCachePodLocalSpec{ - Server: &cachev1alpha1.LMCachePodLocalServerSpec{ - Image: testMPServerImage, - Port: 6555, - L1Capacity: l1, - MaxWorkers: 1, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), - corev1.ResourceMemory: resource.MustParse("2Gi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("3Gi"), - }, - }, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("4Gi"), + MaxWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, }, - }, + }}, }, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}, }, } } -func TestValidatorMPProviderMatrix(t *testing.T) { - tests := []struct { - name string - runtime cachev1alpha1.CacheBackendRuntime - provider cachev1alpha1.CacheBackendRemoteStorageProvider - wantErr bool - }{ - {name: "vLLM host-only", runtime: cachev1alpha1.CacheBackendRuntimeVLLM}, - {name: "SGLang host-only", runtime: cachev1alpha1.CacheBackendRuntimeSGLang}, - {name: "vLLM Redis", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis}, - {name: "SGLang Redis", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis}, - {name: "vLLM legacy LMCacheServer", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, wantErr: true}, - {name: "SGLang legacy LMCacheServer", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, wantErr: true}, - {name: "vLLM Mooncake future", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, wantErr: true}, - {name: "SGLang Mooncake future", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, wantErr: true}, - } - - validator := shippingValidator() - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cb := validPodLocalMPBackend() - cb.Name = "mp" - cb.Namespace = "default" - cb.Spec.Runtime = tc.runtime - if tc.provider != "" { - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: tc.provider, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "storage.example:6379", - } - } - _, err := validator.ValidateCreate(context.Background(), cb) - if tc.wantErr && err == nil { - t.Fatal("expected admission rejection") - } - if !tc.wantErr && err != nil { - t.Fatalf("unexpected admission rejection: %v", err) - } - }) - } -} - -func TestValidateLMCacheTopology(t *testing.T) { - tests := []struct { - name string - mutate func(*cachev1alpha1.CacheBackend) - wantField string - }{ - {name: "PodLocal host-only"}, - { - name: "PodLocal Redis", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "redis.example:6379", - } - }, - }, - { - name: "block without topology", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.Topology = "" - }, - wantField: "spec.lmCache.topology", - }, - { - name: "PodLocal missing block", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal = nil - }, - wantField: "spec.lmCache.podLocal", - }, - { - name: "PodLocal and NodeLocal mixed", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} - }, - wantField: "spec.lmCache.nodeLocal", - }, - { - name: "NodeLocal reserved", - mutate: func(cb *cachev1alpha1.CacheBackend) { - server := cb.Spec.LMCache.PodLocal.Server - cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal - cb.Spec.LMCache.PodLocal = nil - cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ - Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ - Image: server.Image, - Port: server.Port, - L1Capacity: server.L1Capacity, - MaxGPUWorkers: 1, - MaxCPUWorkers: 1, - Resources: server.Resources, - }, - } - }, - wantField: "spec.lmCache.topology", - }, - { - name: "legacy host memory mixed", - mutate: func(cb *cachev1alpha1.CacheBackend) { - capacity := resource.MustParse("1Gi") - cb.Spec.LMCache.HostMemory = &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &capacity} - }, - wantField: "spec.lmCache.hostMemory", - }, - { - name: "legacy worker image mixed", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.WorkerImage = "legacy:latest" - }, - wantField: "spec.lmCache.workerImage", - }, - { - name: "legacy serde mixed", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.RemoteSerde = "cachegen" - }, - wantField: "spec.lmCache.remoteSerde", - }, - { - name: "legacy LMCacheServer L3", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "cache.example:8200", - } - }, - wantField: "spec.remoteStorage.provider", - }, - { - name: "Mooncake L3 not implemented", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "mooncake.example:50051", - } - }, - wantField: "spec.remoteStorage.provider", - }, - { - name: "image tag is not immutable", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Image = "lmcache/standalone:v0.5.3" - }, - wantField: "spec.lmCache.podLocal.server.image", - }, - { - name: "digest without image name", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Image = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - }, - wantField: "spec.lmCache.podLocal.server.image", - }, - { - name: "event port collision", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Port = lmcacheKVEventPort - }, - wantField: "spec.lmCache.podLocal.server.port", - }, - { - name: "HTTP health port collision", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Port = lmcacheMPHTTPPort - }, - wantField: "spec.lmCache.podLocal.server.port", - }, - { - name: "memory request below shm budget", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1536Mi") - }, - wantField: "spec.lmCache.podLocal.server.resources.requests[memory]", - }, - { - name: "memory limit below shm budget", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("1536Mi") - }, - wantField: "spec.lmCache.podLocal.server.resources.limits[memory]", - }, - { - name: "memory request and limit equal shm budget", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("2Gi") - cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("2Gi") - }, - }, - { - name: "fractional extended resource", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") - cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") - }, - wantField: "spec.lmCache.podLocal.server.resources.requests[example.com/device]", - }, - { - name: "EventsOnly cannot carry MP", - mutate: func(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly} - }, - wantField: "spec.lmCache", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cb := validPodLocalMPBackend() - if tc.mutate != nil { - tc.mutate(cb) - } - errs := validateLMCacheTopology(cb) - if tc.wantField == "" { - if len(errs) != 0 { - t.Fatalf("unexpected errors: %v", errs) - } - return - } - for _, err := range errs { - if err.Field == tc.wantField { - return - } - } - t.Fatalf("errors %v do not contain field %q", errs, tc.wantField) - }) - } -} - -func TestValidateLMCacheTopologyLeavesLegacyShapeUntouched(t *testing.T) { - cb := validPodLocalMPBackend() - cb.Spec.LMCache.Topology = "" - cb.Spec.LMCache.PodLocal = nil - cb.Spec.LMCache.WorkerImage = "legacy-worker:test" - if errs := validateLMCacheTopology(cb); len(errs) != 0 { - t.Fatalf("legacy topology-less shape should be left to compatibility rules: %v", errs) - } -} - func TestRejectUnimplementedRedisBindingFeatures(t *testing.T) { cb := validPodLocalMPBackend() cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ @@ -421,3 +174,105 @@ func TestValidateRedisAuthenticationForTypedPodLocal(t *testing.T) { } }) } + +func TestValidateLMCacheTopologyRequiresTypedPodLocal(t *testing.T) { + cb := validPodLocalMPBackend() + if errs := validateLMCacheTopology(cb); len(errs) != 0 { + t.Fatalf("valid typed PodLocal errors: %v", errs) + } + + cb.Spec.LMCache.Topology = "" + if errs := validateLMCacheTopology(cb); len(errs) == 0 { + t.Fatal("topology-less LMCache was accepted") + } +} + +func TestValidateLMCacheTopologyRejectsNodeLocalUntilImplemented(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} + if errs := validateLMCacheTopology(cb); len(errs) == 0 { + t.Fatal("NodeLocal was accepted before Phase 8") + } +} + +func TestValidateLMCacheTopologyRejectsUnpinnedImage(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.PodLocal.Server.Image = "registry.example/lmcache:latest" + if errs := validateLMCacheTopology(cb); len(errs) == 0 { + t.Fatal("mutable MP server image was accepted") + } +} + +func TestValidateLMCacheTopologyRejectsInsufficientMemory(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") + if errs := validateLMCacheTopology(cb); len(errs) == 0 { + t.Fatal("MP server memory below L1 plus headroom was accepted") + } +} + +func TestValidateLMCacheTopologyCurrentMatrix(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend) + wantField string + }{ + {name: "host only"}, + {name: "external Redis", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, + Endpoint: "redis.example:6379", Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + }}, + {name: "missing PodLocal block", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal = nil }, wantField: "spec.lmCache.podLocal"}, + {name: "mixed NodeLocal block", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} + }, wantField: "spec.lmCache.nodeLocal"}, + {name: "NodeLocal reserved", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} + }, wantField: "spec.lmCache.topology"}, + {name: "digest without repository", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Image = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, wantField: "spec.lmCache.podLocal.server.image"}, + {name: "KV event port collision", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Port = lmcacheKVEventPort }, wantField: "spec.lmCache.podLocal.server.port"}, + {name: "HTTP health port collision", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Port = lmcacheMPHTTPPort }, wantField: "spec.lmCache.podLocal.server.port"}, + {name: "memory request below shm", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("4Gi") + }, wantField: "spec.lmCache.podLocal.server.resources.requests[memory]"}, + {name: "memory limit below shm", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceMemory] = resource.MustParse("4Gi") + }, wantField: "spec.lmCache.podLocal.server.resources.limits[memory]"}, + {name: "fractional extended resource", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") + cb.Spec.LMCache.PodLocal.Server.Resources.Limits[corev1.ResourceName("example.com/device")] = resource.MustParse("500m") + }, wantField: "spec.lmCache.podLocal.server.resources.requests[example.com/device]"}, + {name: "EventsOnly cannot carry MP", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly + }, wantField: "spec.lmCache"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := validPodLocalMPBackend() + if tc.mutate != nil { + tc.mutate(cb) + } + errs := validateLMCacheTopology(cb) + if tc.wantField == "" { + if len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + return + } + for _, err := range errs { + if err.Field == tc.wantField { + return + } + } + t.Fatalf("errors %v do not contain field %q", errs, tc.wantField) + }) + } +} diff --git a/internal/webhook/v1alpha1/cachebackend_override_validation_test.go b/internal/webhook/v1alpha1/cachebackend_override_validation_test.go index 42839292..5dcb5286 100644 --- a/internal/webhook/v1alpha1/cachebackend_override_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_override_validation_test.go @@ -6,17 +6,39 @@ package v1alpha1 import ( "context" + "testing" + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" corev1 "k8s.io/api/core/v1" - "strings" - "testing" ) +func TestValidatorRejectsTypedMPReservedOverride(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.Integration.EngineOverrides = &cachev1alpha1.EngineInjectionOverrides{ + SuppressArgs: []string{"--disable-hybrid-kv-cache-manager"}, + } + requireInvalidWithCause(t, shippingValidator(), cb, + "spec.integration.engineOverrides.suppressArgs[0]", "--disable-hybrid-kv-cache-manager") +} + +func withVLLMOverrides(overrides cachev1alpha1.EngineInjectionOverrides) *cachev1alpha1.CacheBackend { + backend := validPodLocalMPBackend() + backend.Spec.Integration.EngineOverrides = &overrides + return backend +} + +func withSGLangOverrides(overrides cachev1alpha1.EngineInjectionOverrides) *cachev1alpha1.CacheBackend { + backend := validPodLocalMPBackend() + backend.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang + backend.Spec.Integration.EngineOverrides = &overrides + return backend +} + func TestValidator_EngineOverrides_NoOverrideAdmitted(t *testing.T) { // Sanity baseline: a CacheBackend whose integration is set but carries // no engineOverrides block must admit unchanged. Locked decision #7 // (byte-identical default) hinges on this. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := newBackend() cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} @@ -26,7 +48,7 @@ func TestValidator_EngineOverrides_NoOverrideAdmitted(t *testing.T) { } func TestValidator_EngineOverrides_SuppressReservedArgRejected(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ SuppressArgs: []string{"--kv-transfer-config"}, }) @@ -40,22 +62,8 @@ func TestValidator_EngineOverrides_SuppressReservedArgRejected(t *testing.T) { "spec.integration.engineOverrides.suppressArgs[0]", "\"vllm\"") } -func TestValidator_TypedVLLMMPReservedSurfaceRejected(t *testing.T) { - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := validPodLocalMPBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - EngineOverrides: &cachev1alpha1.EngineInjectionOverrides{ - SuppressArgs: []string{"--disable-hybrid-kv-cache-manager"}, - }, - } - requireInvalidWithCause(t, v, cb, - "spec.integration.engineOverrides.suppressArgs[0]", - "--disable-hybrid-kv-cache-manager") -} - func TestValidator_EngineOverrides_OverrideReservedArgRejected(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() // Two forms: bare flag and equals form. Both must trip the rule, since // both express the same leading flag token. for _, form := range []string{"--kv-transfer-config", "--kv-transfer-config=alt"} { @@ -68,31 +76,11 @@ func TestValidator_EngineOverrides_OverrideReservedArgRejected(t *testing.T) { } } -func TestValidator_EngineOverrides_SuppressReservedEnvRejected(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - SuppressEnv: []string{"VLLM_USE_V1"}, - }) - requireInvalidWithCause(t, v, cb, - "spec.integration.engineOverrides.suppressEnv[0]", - "VLLM_USE_V1") -} - -func TestValidator_EngineOverrides_OverrideReservedEnvRejected(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - Env: []corev1.EnvVar{{Name: "INFERENCECACHE_FAIL_OPEN", Value: "false"}}, - }) - requireInvalidWithCause(t, v, cb, - "spec.integration.engineOverrides.env[0].name", - "INFERENCECACHE_FAIL_OPEN") -} - func TestValidator_EngineOverrides_OverridePythonHashSeedRejected(t *testing.T) { // PYTHONHASHSEED is reserved (the deterministic-NONE_HASH correctness // invariant). An operator override must be hard-rejected, not silently // applied — re-randomizing the seed 0-hits LMCache reload under TP>1. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{Name: "PYTHONHASHSEED", Value: "1"}}, }) @@ -104,7 +92,7 @@ func TestValidator_EngineOverrides_OverridePythonHashSeedRejected(t *testing.T) func TestValidator_EngineOverrides_SuppressPythonHashSeedRejected(t *testing.T) { // ...and suppression is equally rejected: the operator must not be able to // drop the invariant either. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ SuppressEnv: []string{"PYTHONHASHSEED"}, }) @@ -169,7 +157,7 @@ func TestValidator_EngineOverrides_NonReservedAdmitted(t *testing.T) { // suppresses a flag the adapter wouldn't inject anyway (no-op) and // adds a perf knob. We pin the happy path here so a future tightening // of the rule doesn't accidentally reject legitimate overrides. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Args: []string{"--max-model-len", "8192"}, SuppressArgs: []string{"--enforce-eager"}, @@ -188,7 +176,7 @@ func TestValidator_EngineOverrides_PositionalArgIgnored(t *testing.T) { // because the merge classifies them differently. Admission must treat // them the same way and not surface a spurious rejection — the engine // would happily accept the positional, so admission must too. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Args: []string{"some-positional"}, }) @@ -198,7 +186,7 @@ func TestValidator_EngineOverrides_PositionalArgIgnored(t *testing.T) { } func TestValidator_EngineOverrides_RejectsEmptyEnvName(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{Name: "", Value: "x"}}, }) @@ -208,7 +196,7 @@ func TestValidator_EngineOverrides_RejectsEmptyEnvName(t *testing.T) { } func TestValidator_EngineOverrides_RejectsInvalidEnvName(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ // "=" is forbidden in K8s env var names. Env: []corev1.EnvVar{{Name: "FOO=BAR", Value: "x"}}, @@ -219,7 +207,7 @@ func TestValidator_EngineOverrides_RejectsInvalidEnvName(t *testing.T) { } func TestValidator_EngineOverrides_RejectsValueAndValueFrom(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{ Name: "BOTH", @@ -237,7 +225,7 @@ func TestValidator_EngineOverrides_RejectsValueAndValueFrom(t *testing.T) { func TestValidator_EngineOverrides_RejectsEmptyValueFrom(t *testing.T) { // valueFrom with zero sources fails K8s Pod validation; admission // must catch it before it reaches engine pods. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{ Name: "BAD", @@ -250,7 +238,7 @@ func TestValidator_EngineOverrides_RejectsEmptyValueFrom(t *testing.T) { } func TestValidator_EngineOverrides_RejectsMultipleValueFromSources(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{ Name: "BAD", @@ -314,7 +302,7 @@ func TestEnvVarSourceCount_CountsAllNonNilPointerFields(t *testing.T) { func TestValidator_EngineOverrides_ValueFromAloneAdmitted(t *testing.T) { // Positive case: a ValueFrom-only entry (no Value) is a valid K8s env // shape and must pass. - v := &CacheBackendValidator{Registry: stubRegistry()} + v := shippingValidator() cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ Env: []corev1.EnvVar{{ Name: "POD_NAME", @@ -328,105 +316,12 @@ func TestValidator_EngineOverrides_ValueFromAloneAdmitted(t *testing.T) { } } -func TestValidator_EngineOverrides_ExternalBackendChecksReservedSet(t *testing.T) { - // engineOverrides on an externally owned binding is structurally meaningful: - // the same canonical - // LMCache wire reaches the engine pod whether the cache is managed - // or operator-supplied, so suppressing `--kv-transfer-config` would - // silently un-wire the integration in both cases. The - // reserved-args/env check must therefore fire regardless of ownership. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - SuppressArgs: []string{"--kv-transfer-config"}, - }) - setCanonicalExternalStorage(cb, "shared.team-a.svc.cluster.local:9000") - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("External CR suppressing --kv-transfer-config admitted; reserved-arg check must fire on External too") - } - if !strings.Contains(err.Error(), "--kv-transfer-config") { - t.Fatalf("reserved-arg rejection should name the offending flag; got %v", err) - } -} - -func TestValidator_EngineOverrides_MooncakeBackendChecksReservedSet(t *testing.T) { - // A Mooncake binding reuses the LMCache connector wire (pointed at a - // mooncakestore:// remote), so the same runtime adapter declares the same - // reserved args/env. An operator must not be able to - // un-wire it via engineOverrides any more than on LMCache/External. Use the - // explicitly injected built-in shipping registry so the shipping - // adapter's ReservedArgs/ReservedEnv drive the admission check. - v := shippingValidator() - - // Arg side: suppressing the connector arg must hard-reject, naming the flag. - cbArg := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - SuppressArgs: []string{"--kv-transfer-config"}, - }) - setCanonicalMooncakeStorage(cbArg) - if _, err := v.ValidateCreate(context.Background(), cbArg); err == nil || - !strings.Contains(err.Error(), "--kv-transfer-config") { - t.Fatalf("Mooncake CR suppressing --kv-transfer-config must reject naming the flag; got %v", err) - } - - // Env side: overriding the reserved remote-URL env must hard-reject too. - cbEnv := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - Env: []corev1.EnvVar{{Name: "LMCACHE_REMOTE_URL", Value: "mooncakestore://evil:50051"}}, - }) - setCanonicalMooncakeStorage(cbEnv) - if _, err := v.ValidateCreate(context.Background(), cbEnv); err == nil || - !strings.Contains(err.Error(), "LMCACHE_REMOTE_URL") { - t.Fatalf("Mooncake CR overriding reserved LMCACHE_REMOTE_URL must reject naming the env; got %v", err) - } -} - -func TestValidator_EngineOverrides_NilRegistry_FallsBackToShippingSet(t *testing.T) { - // A zero-value validator (Registry: nil) must consult the SAME - // shipping adapter set in BOTH checkRuntimeAdapter and - // checkEngineOverrides — otherwise an external binding could admit and then - // silently bypass reserved-arg enforcement, letting an - // operator un-wire the cache at the engine pod. Pin both halves of - // the contract: nil-registry rejects External + suppressed - // --kv-transfer-config with a field-scoped error. - v := shippingValidator() - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - SuppressArgs: []string{"--kv-transfer-config"}, - }) - setCanonicalExternalStorage(cb, "shared.team-a.svc.cluster.local:9000") - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("nil-registry validator admitted External + suppressed --kv-transfer-config; reserved-arg check must fire via the shipping-set fallback") - } - if !strings.Contains(err.Error(), "--kv-transfer-config") { - t.Fatalf("expected rejection naming the offending flag; got %v", err) +func TestValidatorAdmitsNonReservedOverride(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.Integration.EngineOverrides = &cachev1alpha1.EngineInjectionOverrides{ + Args: []string{"--max-model-len=4096"}, } -} - -func TestValidator_EngineOverrides_ExternalBackendAdmittedWhenSafe(t *testing.T) { - // An externally owned CR carrying engineOverrides that DON'T touch the - // adapter's reserved set must still admit. The LMCache wire is shared across - // ownership modes. LMCACHE_CHUNK_SIZE - // is a perf knob, not reserved; suppressing or amending it is fine. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - Env: []corev1.EnvVar{{Name: "LMCACHE_CHUNK_SIZE", Value: "512"}}, - }) - setCanonicalExternalStorage(cb, "shared.team-a.svc.cluster.local:9000") - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("External CR with non-reserved override rejected: %v", err) + if _, err := shippingValidator().ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("non-reserved override rejected: %v", err) } } - -func TestValidator_EngineOverrides_ExternalRejectsPythonHashSeedOverride(t *testing.T) { - // The shared LMCache runtime adapter reserves the same env across ownership - // modes, so a PYTHONHASHSEED override on an externally owned CR - // is hard-rejected for the same reason — proving the correctness - // invariant holds across both ownership modes, not just managed. - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := withVLLMOverrides(cachev1alpha1.EngineInjectionOverrides{ - Env: []corev1.EnvVar{{Name: "PYTHONHASHSEED", Value: "1"}}, - }) - setCanonicalExternalStorage(cb, "shared.team-a.svc.cluster.local:9000") - requireInvalidWithCause(t, v, cb, - "spec.integration.engineOverrides.env[0].name", - "PYTHONHASHSEED") -} diff --git a/internal/webhook/v1alpha1/cachebackend_storage_validation.go b/internal/webhook/v1alpha1/cachebackend_storage_validation.go index f65fa91e..fe55b866 100644 --- a/internal/webhook/v1alpha1/cachebackend_storage_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_storage_validation.go @@ -15,22 +15,6 @@ import ( "strings" ) -func rejectNonPositiveHostMemoryCapacity(cb *cachev1alpha1.CacheBackend) field.ErrorList { - if cb.Spec.LMCache == nil || cb.Spec.LMCache.HostMemory == nil || - cb.Spec.LMCache.HostMemory.Capacity == nil || - cb.Spec.LMCache.HostMemory.Capacity.Sign() > 0 { - return nil - } - - return field.ErrorList{ - field.Invalid( - field.NewPath("spec", "lmCache", "hostMemory", "capacity"), - cb.Spec.LMCache.HostMemory.Capacity.String(), - "must be greater than zero", - ), - } -} - func selectedProviderResources(cb *cachev1alpha1.CacheBackend) (*corev1.ResourceRequirements, *field.Path) { if cb == nil || cb.Spec.RemoteStorage == nil { return nil, nil @@ -42,14 +26,6 @@ func selectedProviderResources(cb *cachev1alpha1.CacheBackend) (*corev1.Resource if storage.Redis != nil { return storage.Redis.Resources, storagePath.Child("redis", "resources") } - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - if storage.LMCacheServer != nil { - return storage.LMCacheServer.Resources, storagePath.Child("lmCacheServer", "resources") - } - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if storage.Mooncake != nil { - return storage.Mooncake.Resources, storagePath.Child("mooncake", "resources") - } } return nil, nil } @@ -58,13 +34,6 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { var errs field.ErrorList specPath := field.NewPath("spec") - if cb.Spec.RemoteStorage == nil && cb.Spec.Autoscaling != nil { - errs = append(errs, field.Forbidden( - specPath.Child("autoscaling"), - "host-only backends omit spec.remoteStorage and provision no provider workload, so there is nothing to autoscale", - )) - } - if cb.Spec.LMCache != nil && cb.Spec.EffectiveCacheType() != cachev1alpha1.CacheBackendTypeLMCache { errs = append(errs, field.Forbidden( specPath.Child("lmCache"), @@ -87,7 +56,7 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { case cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged: if strings.TrimSpace(storage.Endpoint) != "" { errs = append(errs, field.Forbidden(storagePath.Child("endpoint"), - "managed providers publish their observed endpoint in status.endpoint")) + "managed providers publish their observed endpoint in status.remoteStorage.endpoint")) } case cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal: if strings.TrimSpace(storage.Endpoint) == "" { @@ -96,6 +65,10 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { } else if err := backendadapter.ValidateExternalEndpoint(storage.Provider, storage.Endpoint); err != nil { errs = append(errs, field.Invalid(storagePath.Child("endpoint"), storage.Endpoint, err.Error())) } + if storage.Workload != nil { + errs = append(errs, field.Forbidden(storagePath.Child("workload"), + "valid only with Managed ownership")) + } } type providerConfig struct { @@ -103,51 +76,18 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { set bool path *field.Path } - configs := []providerConfig{ - {cachev1alpha1.CacheBackendRemoteStorageProviderRedis, storage.Redis != nil, storagePath.Child("redis")}, - {cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, storage.LMCacheServer != nil, storagePath.Child("lmCacheServer")}, - {cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, storage.Mooncake != nil, storagePath.Child("mooncake")}, - } + configs := []providerConfig{{cachev1alpha1.CacheBackendRemoteStorageProviderRedis, storage.Redis != nil, storagePath.Child("redis")}} for _, config := range configs { if config.set && storage.Provider != config.provider { errs = append(errs, field.Forbidden(config.path, fmt.Sprintf("configuration belongs to provider %s, but remoteStorage.provider=%s", config.provider, storage.Provider))) } if config.set && storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal { - if config.provider == cachev1alpha1.CacheBackendRemoteStorageProviderRedis { - // Redis combines connection settings (authentication/TLS/database), - // which apply to either ownership mode, with managed-workload - // settings. External bindings may retain the former but cannot ask - // this controller to choose an image or container resources. - if strings.TrimSpace(storage.Redis.Image) != "" { - errs = append(errs, field.Forbidden(config.path.Child("image"), - "valid only with Managed ownership")) - } - if storage.Redis.Resources != nil { - errs = append(errs, field.Forbidden(config.path.Child("resources"), - "valid only with Managed ownership")) - } - continue - } - errs = append(errs, field.Forbidden(config.path, - "provider workload configuration is valid only with Managed ownership")) - } - } - if storage.Ownership == cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged { - switch storage.Provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - if storage.LMCacheServer != nil { - errs = append(errs, validateManagedProviderCommand( - storagePath.Child("lmCacheServer", "command"), - storage.LMCacheServer.Command, - )...) + if strings.TrimSpace(storage.Redis.Image) != "" { + errs = append(errs, field.Forbidden(config.path.Child("image"), "valid only with Managed ownership")) } - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if storage.Mooncake != nil { - errs = append(errs, validateManagedProviderCommand( - storagePath.Child("mooncake", "command"), - storage.Mooncake.Command, - )...) + if storage.Redis.Resources != nil { + errs = append(errs, field.Forbidden(config.path.Child("resources"), "valid only with Managed ownership")) } } } @@ -155,23 +95,6 @@ func validateCacheHierarchy(cb *cachev1alpha1.CacheBackend) field.ErrorList { return errs } -func validateManagedProviderCommand(path *field.Path, command []string) field.ErrorList { - if command == nil { - return nil - } - if len(command) == 0 { - return field.ErrorList{field.Invalid(path, command, "must contain an executable")} - } - - var errs field.ErrorList - for i, part := range command { - if strings.TrimSpace(part) == "" { - errs = append(errs, field.Invalid(path.Index(i), part, "must not be empty")) - } - } - return errs -} - // rejectCrossNamespaceEndpointWithoutOptIn rejects an external endpoint that // resolves into a Service in a namespace other than the CacheBackend's // own, unless spec.allowCrossNamespace is true. Crossing a namespace is diff --git a/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go b/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go index 9f264be3..ad774cbf 100644 --- a/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_storage_validation_test.go @@ -7,242 +7,55 @@ package v1alpha1 import ( "context" "fmt" + "strings" + "testing" + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "strings" - "testing" ) -func TestValidator_CanonicalCacheHierarchy(t *testing.T) { - validator := shippingValidator() - - t.Run("sglang host-only", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("ValidateCreate: %v", err) - } - }) - - t.Run("host-only rejects autoscaling", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - requireInvalidWithCause(t, validator, cb, "spec.autoscaling", - "host-only backends") - }) - - t.Run("host memory capacity must be positive", func(t *testing.T) { - for _, capacity := range []string{"0", "-1Gi"} { - t.Run(capacity, func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - quantity := resource.MustParse(capacity) - cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ - HostMemory: &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &quantity}, - } - requireInvalidWithCause(t, validator, cb, "spec.lmCache.hostMemory.capacity", - "must be greater than zero") - }) - } - }) - - t.Run("positive host memory capacity is admitted", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - quantity := resource.MustParse("1Gi") - cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ - HostMemory: &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &quantity}, - } - if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("ValidateCreate: %v", err) - } - }) - - t.Run("sglang managed redis", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{Image: "redis:test"}, - } - if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("ValidateCreate: %v", err) - } - }) - - t.Run("vllm rejects resp binding", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - } - _, err := validator.ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), "does not accept") { - t.Fatalf("ValidateCreate error = %v, want binding compatibility rejection", err) - } - }) - - t.Run("external requires endpoint", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - } - _, err := validator.ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), "remoteStorage.endpoint") { - t.Fatalf("ValidateCreate error = %v, want endpoint rejection", err) - } - }) - - t.Run("external endpoint scheme follows provider protocol", func(t *testing.T) { - tests := []struct { - name string - runtime cachev1alpha1.CacheBackendRuntime - provider cachev1alpha1.CacheBackendRemoteStorageProvider - endpoint string - wantErr bool - }{ - {name: "redis bare", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:6379"}, - {name: "redis rejects lm scheme", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "lm://redis.example:6379", wantErr: true}, - {name: "redis rejects named port", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:redis", wantErr: true}, - {name: "redis rejects zero port", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:0", wantErr: true}, - {name: "redis rejects out-of-range port", runtime: cachev1alpha1.CacheBackendRuntimeSGLang, provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, endpoint: "redis.example:70000", wantErr: true}, - {name: "lmcache bare", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:8200"}, - {name: "lmcache explicit scheme", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "lm://cache.example:8200"}, - {name: "lmcache rejects mooncake scheme", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "mooncakestore://cache.example:50051", wantErr: true}, - {name: "lmcache rejects named port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:not-a-port", wantErr: true}, - {name: "lmcache rejects zero port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:0", wantErr: true}, - {name: "lmcache rejects out-of-range port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, endpoint: "cache.example:70000", wantErr: true}, - {name: "mooncake bare", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncake.example:50051"}, - {name: "mooncake explicit scheme", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://mooncake.example:50051"}, - {name: "mooncake rejects lm scheme", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "lm://mooncake.example:50051", wantErr: true}, - {name: "mooncake rejects named port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://mooncake.example:not-a-port", wantErr: true}, - {name: "mooncake rejects zero port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://mooncake.example:0", wantErr: true}, - {name: "mooncake rejects out-of-range port", runtime: cachev1alpha1.CacheBackendRuntimeVLLM, provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, endpoint: "mooncakestore://mooncake.example:70000", wantErr: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = tt.runtime - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: tt.provider, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: tt.endpoint, - } - if tt.wantErr { - requireInvalidWithCause(t, validator, cb, "spec.remoteStorage.endpoint", "") - return - } - if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("ValidateCreate: %v", err) - } - }) - } - }) - - t.Run("external Redis separates binding from managed workload settings", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "redis.example:6379", - Redis: &cachev1alpha1.RedisRemoteStorageSpec{ - Authentication: &cachev1alpha1.RedisAuthenticationSpec{ - Password: corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{Name: "redis-auth"}, - Key: "password", - }, - }, - }, - } - errs := validateCacheHierarchy(cb) - if len(errs) != 0 { - t.Fatalf("external Redis connection settings should be structurally valid: %v", errs) - } - - cb.Spec.RemoteStorage.Redis.Image = "redis:test" - errs = validateCacheHierarchy(cb) - if len(errs) != 1 || errs[0].Field != "spec.remoteStorage.redis.image" { - t.Fatalf("external Redis image errors = %v, want field-scoped managed-setting rejection", errs) - } - }) - - t.Run("managed provider resources are validated at their typed path", func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Redis: &cachev1alpha1.RedisRemoteStorageSpec{ - Resources: &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("-1Gi"), - }, - }, - }, - } - _, err := validator.ValidateCreate(context.Background(), cb) - if err == nil || !strings.Contains(err.Error(), "spec.remoteStorage.redis.resources.limits[memory]") { - t.Fatalf("ValidateCreate error = %v, want typed provider resource path", err) - } - }) - - t.Run("managed provider command requires non-empty entries", func(t *testing.T) { - tests := []struct { - name string - command []string - path string - }{ - {name: "empty command", command: []string{}, path: "spec.remoteStorage.lmCacheServer.command"}, - {name: "empty executable", command: []string{""}, path: "spec.remoteStorage.lmCacheServer.command[0]"}, - {name: "blank argument", command: []string{"cache-server", " "}, path: "spec.remoteStorage.lmCacheServer.command[1]"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - LMCacheServer: &cachev1alpha1.LMCacheServerRemoteStorageSpec{ - Command: tt.command, - }, +func TestValidateCacheHierarchyRedisOwnership(t *testing.T) { + tests := []struct { + name string + ownership cachev1alpha1.CacheBackendRemoteStorageOwnership + endpoint string + image string + workload bool + wantErr string + }{ + {name: "managed", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged}, + {name: "managed endpoint", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, endpoint: "redis.example:6379", wantErr: "managed providers"}, + {name: "external", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, endpoint: "redis.example:6379"}, + {name: "external missing endpoint", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, wantErr: "required"}, + {name: "external scheme", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, endpoint: "redis://redis.example:6379", wantErr: "schemes are not supported"}, + {name: "external managed image", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, endpoint: "redis.example:6379", image: "redis:7", wantErr: "Managed ownership"}, + {name: "managed workload", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, workload: true}, + {name: "external managed workload", ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, endpoint: "redis.example:6379", workload: true, wantErr: "Managed ownership"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, + Ownership: tc.ownership, + Endpoint: tc.endpoint, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{Image: tc.image}, + } + if tc.workload { + cb.Spec.RemoteStorage.Workload = &cachev1alpha1.CacheBackendManagedWorkloadSpec{ + NodeSelector: map[string]string{"pool": "cache"}, } - requireInvalidWithCause(t, validator, cb, tt.path, "must") - }) - } - - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - Mooncake: &cachev1alpha1.MooncakeRemoteStorageSpec{ - Command: []string{""}, - }, - } - requireInvalidWithCause(t, validator, cb, "spec.remoteStorage.mooncake.command[0]", "must not be empty") - }) - - t.Run("typed observation does not synthesize provider storage", func(t *testing.T) { - cb := newBackend() - cb.Spec.Observation = &cachev1alpha1.CacheBackendObservationSpec{ModelID: "model-a"} - if _, err := validator.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("ValidateCreate: %v", err) - } - storage := cb.Spec.EffectiveRemoteStorage() - if storage != nil { - t.Fatalf("EffectiveRemoteStorage() = %+v, want nil", storage) - } - }) - + } + errs := validateCacheHierarchy(cb) + if tc.wantErr == "" && len(errs) != 0 { + t.Fatalf("unexpected errors: %v", errs) + } + if tc.wantErr != "" && !strings.Contains(errs.ToAggregate().Error(), tc.wantErr) { + t.Fatalf("errors = %v, want %q", errs, tc.wantErr) + } + }) + } } func TestValidator_ResourcesLimitsBelowRequestsRejected(t *testing.T) { @@ -254,7 +67,7 @@ func TestValidator_ResourcesLimitsBelowRequestsRejected(t *testing.T) { // diagnose through downstream kubectl-describe spelunking). v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("8Gi"), }, @@ -262,8 +75,8 @@ func TestValidator_ResourcesLimitsBelowRequestsRejected(t *testing.T) { corev1.ResourceMemory: resource.MustParse("4Gi"), }, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.limits[memory]", - "must be greater than or equal to spec.remoteStorage.lmCacheServer.resources.requests[memory]") + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.limits[memory]", + "must be greater than or equal to spec.remoteStorage.redis.resources.requests[memory]") } func TestValidator_ResourcesLimitsEqualRequestsAdmitted(t *testing.T) { @@ -271,7 +84,7 @@ func TestValidator_ResourcesLimitsEqualRequestsAdmitted(t *testing.T) { // admit. The rule only rejects strict-less-than. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, } @@ -285,7 +98,7 @@ func TestValidator_ResourcesRequestsOnlyAdmitted(t *testing.T) { // rule MUST NOT synthesise a phantom limit to compare against. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, } if _, err := v.ValidateCreate(context.Background(), cb); err != nil { @@ -299,7 +112,7 @@ func TestValidator_ResourcesLimitsOnlyAdmitted(t *testing.T) { // rule must not fire. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("8Gi")}, } if _, err := v.ValidateCreate(context.Background(), cb); err != nil { @@ -318,7 +131,7 @@ func TestValidator_ResourcesFractionalExtendedRejected(t *testing.T) { t.Run(side, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{} + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{} entry := corev1.ResourceList{ corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("500m"), } @@ -326,14 +139,14 @@ func TestValidator_ResourcesFractionalExtendedRejected(t *testing.T) { corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("500m"), } if side == "requests" { - managedLMCacheServer(cb).Resources.Requests = entry - managedLMCacheServer(cb).Resources.Limits = matching + cb.Spec.RemoteStorage.Redis.Resources.Requests = entry + cb.Spec.RemoteStorage.Redis.Resources.Limits = matching } else { - managedLMCacheServer(cb).Resources.Limits = entry - managedLMCacheServer(cb).Resources.Requests = matching + cb.Spec.RemoteStorage.Redis.Resources.Limits = entry + cb.Spec.RemoteStorage.Redis.Resources.Requests = matching } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.%s[nvidia.com/gpu]", side), + fmt.Sprintf("spec.remoteStorage.redis.resources.%s[nvidia.com/gpu]", side), "must be an integer quantity") }) } @@ -344,7 +157,7 @@ func TestValidator_ResourcesIntegerExtendedAdmitted(t *testing.T) { // admit — the rule fires only on fractional values. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), }, @@ -363,7 +176,7 @@ func TestValidator_ResourcesFractionalCPUAdmitted(t *testing.T) { // only to vendor-prefixed extended resources. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("250m")}, } if _, err := v.ValidateCreate(context.Background(), cb); err != nil { @@ -389,12 +202,12 @@ func TestValidator_ResourcesRequestsOnlyNonOvercommittableRejected(t *testing.T) if name == "hugepages-2Mi" { qty = "2Mi" } - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{name: resource.MustParse(qty)}, } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.requests[%s]", name), - "must also be set in spec.remoteStorage.lmCacheServer.resources.limits") + fmt.Sprintf("spec.remoteStorage.redis.resources.requests[%s]", name), + "must also be set in spec.remoteStorage.redis.resources.limits") }) } } @@ -405,7 +218,7 @@ func TestValidator_ResourcesLimitsOnlyNonOvercommittableAdmitted(t *testing.T) { // The rule we add fires only on the requests-only direction. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), }, @@ -423,7 +236,7 @@ func TestValidator_ResourcesRequestsOnlyOvercommittableAdmitted(t *testing.T) { t.Run(string(name), func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{name: resource.MustParse("1")}, } if _, err := v.ValidateCreate(context.Background(), cb); err != nil { @@ -453,13 +266,13 @@ func TestValidator_ResourcesNonOvercommittableMismatchRejected(t *testing.T) { t.Run(tc.name, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{tc.resource: resource.MustParse(tc.req)}, Limits: corev1.ResourceList{tc.resource: resource.MustParse(tc.lim)}, } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.limits[%s]", tc.resource), - "must equal spec.remoteStorage.lmCacheServer.resources.requests") + fmt.Sprintf("spec.remoteStorage.redis.resources.limits[%s]", tc.resource), + "must equal spec.remoteStorage.redis.resources.requests") }) } } @@ -468,7 +281,7 @@ func TestValidator_ResourcesNonOvercommittableEqualAdmitted(t *testing.T) { // The same non-overcommittable resources admit when limits == requests. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, Limits: corev1.ResourceList{corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1")}, } @@ -489,13 +302,13 @@ func TestValidator_ResourcesReservedPrefixesRejected(t *testing.T) { t.Run(name, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName(name): resource.MustParse("1"), }, } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.requests[%s]", name), + fmt.Sprintf("spec.remoteStorage.redis.resources.requests[%s]", name), "not a valid container resource name") }) } @@ -508,12 +321,12 @@ func TestValidator_ResourcesInvalidNameRejected(t *testing.T) { // admission so the regression surfaces at `kubectl apply`. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName("memory!"): resource.MustParse("4Gi"), }, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.requests[memory!]", + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.requests[memory!]", "not a valid container resource name") } @@ -527,12 +340,12 @@ func TestValidator_ResourcesUnqualifiedNonStandardNameRejected(t *testing.T) { // than chasing it through a child Deployment apply. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName("foo"): resource.MustParse("1"), }, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.requests[foo]", + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.requests[foo]", "not a valid container resource name") } @@ -546,13 +359,13 @@ func TestValidator_ResourcesMalformedHugepagesRejected(t *testing.T) { t.Run(name, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName(name): resource.MustParse("1"), }, } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.requests[%s]", name), + fmt.Sprintf("spec.remoteStorage.redis.resources.requests[%s]", name), "not a valid container resource name") }) } @@ -579,7 +392,7 @@ func TestValidator_ResourcesStandardContainerResourceNamesAdmitted(t *testing.T) t.Run(string(tc.name), func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{tc.name: resource.MustParse(tc.qty)}, Limits: corev1.ResourceList{tc.name: resource.MustParse(tc.qty)}, } @@ -607,7 +420,7 @@ func TestValidator_ResourcesHugepagesQuantityMustBeDivisible(t *testing.T) { t.Run(tc.page+"/"+tc.qty, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName(tc.page): resource.MustParse(tc.qty), }, @@ -616,7 +429,7 @@ func TestValidator_ResourcesHugepagesQuantityMustBeDivisible(t *testing.T) { }, } requireInvalidWithCause(t, v, cb, - fmt.Sprintf("spec.remoteStorage.lmCacheServer.resources.requests[%s]", tc.page), + fmt.Sprintf("spec.remoteStorage.redis.resources.requests[%s]", tc.page), "must be a multiple of the page size") }) } @@ -638,7 +451,7 @@ func TestValidator_ResourcesHugepagesAlignedQuantityAdmitted(t *testing.T) { t.Run(tc.page+"/"+tc.qty, func(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceName(tc.page): resource.MustParse(tc.qty), }, @@ -660,7 +473,7 @@ func TestValidator_ResourcesValidExtendedNameAdmitted(t *testing.T) { // structurally forbidden). v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), }, @@ -678,20 +491,20 @@ func TestValidator_ResourcesNegativeRequestRejected(t *testing.T) { // the regression surfaces at `kubectl apply`. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("-1Gi")}, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.requests[memory]", + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.requests[memory]", "must be a non-negative quantity") } func TestValidator_ResourcesNegativeLimitRejected(t *testing.T) { v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("-100m")}, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.limits[cpu]", + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.limits[cpu]", "must be a non-negative quantity") } @@ -702,7 +515,7 @@ func TestValidator_ResourcesZeroQuantityAdmitted(t *testing.T) { // shape. Only strictly-negative quantities are rejected. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("0")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("8Gi")}, } @@ -721,11 +534,11 @@ func TestValidator_ResourcesClaimsRejected(t *testing.T) { // renderer learns to thread resourceClaims onto the PodSpec. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Claims: []corev1.ResourceClaim{{Name: "gpu-claim"}}, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.claims", - "spec.remoteStorage.lmCacheServer.resources.claims is not supported") + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.claims", + "spec.remoteStorage.redis.resources.claims is not supported") } func TestValidator_ResourcesEmptyClaimsAdmitted(t *testing.T) { @@ -733,7 +546,7 @@ func TestValidator_ResourcesEmptyClaimsAdmitted(t *testing.T) { // operator-supplied entries, never on the absence of the field. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, } if _, err := v.ValidateCreate(context.Background(), cb); err != nil { @@ -748,349 +561,22 @@ func TestValidator_ResourcesCPULimitsBelowRequestsRejected(t *testing.T) { // silently narrow the rule back to memory-only. v := shippingValidator() cb := newBackend() - managedLMCacheServer(cb).Resources = &corev1.ResourceRequirements{ + cb.Spec.RemoteStorage.Redis.Resources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("250m")}, } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.lmCacheServer.resources.limits[cpu]", - "must be greater than or equal to spec.remoteStorage.lmCacheServer.resources.requests[cpu]") -} - -func TestValidator_CrossNamespaceEndpointWithoutOptInRejected(t *testing.T) { - v := shippingValidator() - cb := newBackend() - setCanonicalExternalStorage(cb, "shared-cache.team-b.svc.cluster.local:9000") - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "references namespace \"team-b\"") -} - -func TestValidator_CrossNamespaceEndpointWithOptInAdmitted(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistry()} - cb := newBackend() - // Carry a port — the new shape rule requires host:port; the - // cross-namespace assertion below is unaffected by the port suffix. - setCanonicalExternalStorage(cb, "shared-cache.team-b.svc.cluster.local:9000") - cb.Spec.AllowCrossNamespace = true - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("External cross-namespace endpoint with opt-in rejected: %v", err) - } -} - -func TestValidator_CanonicalCrossNamespaceEndpointWithoutOptInRejected(t *testing.T) { - v := shippingValidator() - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "shared-cache.team-b.svc.cluster.local:9000", - } - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "references namespace \"team-b\"") + requireInvalidWithCause(t, v, cb, "spec.remoteStorage.redis.resources.limits[cpu]", + "must be greater than or equal to spec.remoteStorage.redis.resources.requests[cpu]") } -func TestValidator_CanonicalCrossNamespaceEndpointWithOptInAdmitted(t *testing.T) { - v := shippingValidator() - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM +func TestValidateCacheHierarchyRejectsProviderConfigMismatch(t *testing.T) { + cb := validPodLocalMPBackend() cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: "shared-cache.team-b.svc.cluster.local:9000", - } - cb.Spec.AllowCrossNamespace = true - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("canonical cross-namespace endpoint with opt-in rejected: %v", err) + Provider: "removed", + Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } -} - -func TestValidator_ExternalEndpoint_LMSchemeAdmitted(t *testing.T) { - // Operators who prefer to be explicit can pre-fix the endpoint with - // the LMCache lm:// scheme; the adapter passes it through unchanged. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://cache.example.com:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("External with lm:// scheme rejected: %v", err) - } -} - -func TestValidator_ExternalEndpoint_HTTPSchemeRejected(t *testing.T) { - // A non-lm:// scheme would concatenate to LMCACHE_REMOTE_URL=lm:// - // https://... at injection time, which the LMCache connector - // rejects. Catch the misconfiguration at admission instead of in - // engine-pod crash logs. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "https://cache.example.com:443/api") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - `scheme "https" is not supported`) -} - -func TestValidator_ExternalEndpoint_PathRejected(t *testing.T) { - // LMCache is a TCP-level protocol — paths/queries/fragments don't - // belong on the wire and would be silently dropped at the engine - // connector. Reject them at admission so the rejection message - // surfaces the problem. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example.com:8200/path") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be host:port (optionally prefixed lm://)") -} - -func TestValidator_ExternalEndpoint_LMSchemeOnlyRejected(t *testing.T) { - // `lm://` alone is just the scheme — no host. Without this check - // the CR admits, goes Ready=True, and the pod webhook injects - // LMCACHE_REMOTE_URL=lm:// (the exact broken value the validation - // exists to prevent). - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_PortOnlyRejected(t *testing.T) { - // `:8200` is a port with no host — same broken-injection risk. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, ":8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_LMSchemePortOnlyRejected(t *testing.T) { - // Scheme + port with no host. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_PortlessHostRejected(t *testing.T) { - // Bare host with no port is rejected: the LMCache connector dials a - // specific TCP target, so spec.remoteStorage.endpoint must carry both - // halves. - // Without this check the CR admits and the engine boots with - // LMCACHE_REMOTE_URL=lm://cache.example.com — the connector then - // either picks an undocumented default or crashes; either way the - // failure surfaces at the engine, not at admission where it belongs. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example.com") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_EmptyPortRejected(t *testing.T) { - // Trailing colon with no port (`host:`) is the failure mode of an - // operator who started typing the port and saved. Same broken - // LMCACHE_REMOTE_URL=lm://cache.example.com: at injection. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example.com:") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_PortlessLMSchemeRejected(t *testing.T) { - // Same rule applies when the scheme is explicit. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://cache.example.com") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_PortlessIPv6Rejected(t *testing.T) { - // Bracket-only IPv6 (`[::1]`) has no port either — reject for the - // same reason. Validates that the bracket-aware path enforces the - // port-required rule. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "[2001:db8::1]") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_EmbeddedWhitespaceRejected(t *testing.T) { - // Leading/trailing whitespace is already trimmed for friendliness; - // whitespace *inside* the address is not. `cache example:8200` - // would otherwise pass the host:port split (host="cache example", - // port="8200") and inject a malformed LMCACHE_REMOTE_URL — the - // LMCache connector refuses to dial it at engine startup. Catch - // the misconfiguration loudly at write time. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache example.com:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must not contain whitespace or control characters") -} - -func TestValidator_ExternalEndpoint_EmbeddedWhitespaceInPortRejected(t *testing.T) { - // Same rule applies to the port half — `cache.example:82 00` - // would split host="cache.example", port="82 00" and inject a - // broken URL. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example:82 00") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must not contain whitespace or control characters") -} - -func TestValidator_ExternalEndpoint_ControlCharRejected(t *testing.T) { - // Embedded control chars (newline, tab, etc.) are rejected even - // though they're "whitespace": same broken-URL injection risk, - // plus a defence-in-depth against header injection if a future - // consumer ever templates the endpoint into a text format. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example.com:8200\nLMCACHE_LOG_LEVEL=debug") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must not contain whitespace or control characters") -} - -func TestValidator_ExternalEndpoint_BracketedIPv6ExtraColonRejected(t *testing.T) { - // The bracketed form `[::1]:8200:bad` would otherwise pass with - // host="::1" port="8200:bad" — the bracket strips the IPv6 colons - // out of the host/port boundary calculation, but the naive port - // half still contains the trailing `:bad`. Reject: the brackets are - // the contract that makes the boundary unambiguous; sneaking an - // extra colon past them produces an invalid - // LMCACHE_REMOTE_URL=lm://[::1]:8200:bad at injection. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "[::1]:8200:bad") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_BracketedIPv6ExtraColonWithSchemeRejected(t *testing.T) { - // Same bug surface with the explicit scheme — the scheme strip - // shouldn't change the host:port shape check that follows. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://[::1]:8200:bad") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_UnbracketedIPv6Rejected(t *testing.T) { - // RFC 3986 requires brackets for IPv6 in URI authority components, - // and there is no unambiguous host:port boundary without them. A - // naive LastIndex(":") split would treat `2001:db8::1` as host= - // "2001:db8:" port="1" — admission would pass and the engine pod - // would inject LMCACHE_REMOTE_URL=lm://2001:db8::1, which the - // LMCache connector cannot parse. Refuse at write time. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "2001:db8::1") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be a non-empty host AND port") -} - -func TestValidator_ExternalEndpoint_IPv6Admitted(t *testing.T) { - // IPv6 literals require brackets in host:port form; the validator - // must accept them rather than mistaking the inner colons for - // scheme/port separators. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "[2001:db8::1]:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("IPv6 endpoint rejected: %v", err) - } -} - -func TestValidator_ExternalEndpoint_LMSchemeWithPathRejected(t *testing.T) { - // Same concern as the bare-host case; the path-after-scheme variant - // is just as broken. - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "lm://cache.example.com:8200/path") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - requireInvalidWithCause(t, v, cb, "spec.remoteStorage.endpoint", - "must be host:port (optionally prefixed lm://)") -} - -func TestServiceDNSNamespace(t *testing.T) { - cases := []struct { - name string - endpoint string - wantNS string - wantOK bool - }{ - {"bare svc DNS", "cache.team-a.svc", "team-a", true}, - {"cluster.local svc DNS", "cache.team-b.svc.cluster.local", "team-b", true}, - {"svc DNS with port", "cache.team-d.svc.cluster.local:9000", "team-d", true}, - {"https scheme + path", "https://cache.team-e.svc.cluster.local/api", "team-e", true}, - {"grpc scheme", "grpc://cache.team-f.svc:9090", "team-f", true}, - {"pod FQDN bare", "cache-0.cache.team-g.svc", "team-g", true}, - {"pod FQDN cluster.local", "cache-0.cache.team-h.svc.cluster.local", "team-h", true}, - {"pod FQDN with port", "cache-1.cache.team-i.svc.cluster.local:9000", "team-i", true}, - {"FQDN trailing dot", "cache.team-j.svc.cluster.local.", "team-j", true}, - {"FQDN trailing dot bare svc", "cache.team-k.svc.", "team-k", true}, - {"uppercase svc DNS", "Cache.TEAM-L.SVC.cluster.local", "team-l", true}, - {"uppercase pod FQDN trailing dot", "CACHE-0.cache.team-m.SVC.cluster.local.", "team-m", true}, - {"external hostname", "cache.example.com", "", false}, - {"external hostname with port", "cache.example.com:443", "", false}, - {"external with svc-shaped label", "cache.team-b.svc.example.com", "", false}, - {"external svc-shaped label with port", "cache.team-b.svc.example.com:443", "", false}, - {"non-default cluster domain", "cache.team-c.svc.private", "", false}, - {"bare hostname", "cache", "", false}, - {"two-label hostname", "cache.team-a", "", false}, - {"third label not svc", "cache.team-a.cluster", "", false}, - {"ipv4", "10.0.0.5:9000", "", false}, - {"empty", "", "", false}, - {"whitespace", " ", "", false}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - ns, ok := serviceDNSNamespace(tc.endpoint) - if ok != tc.wantOK { - t.Fatalf("ok = %v, want %v", ok, tc.wantOK) - } - if ns != tc.wantNS { - t.Fatalf("ns = %q, want %q", ns, tc.wantNS) - } - }) + if errs := validateCacheHierarchy(cb); len(errs) == 0 { + t.Fatal("provider/config mismatch was accepted") } } diff --git a/internal/webhook/v1alpha1/cachebackend_validator.go b/internal/webhook/v1alpha1/cachebackend_validator.go index 6f8491f6..adafd9aa 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator.go +++ b/internal/webhook/v1alpha1/cachebackend_validator.go @@ -53,14 +53,10 @@ var DefaultValidationRules = []ValidationRule{ validateLMCacheTopology, rejectUnimplementedRedisBindingFeatures, rejectCrossNamespaceEndpointWithoutOptIn, - requireExplicitMinReplicasOnScaleToZeroWithAutoscaling, - rejectMooncakeMasterScaleOut, - rejectEngineHostNetworkOnBackendThatDoesNotNeedIt, rejectResourceLimitsBelowRequests, rejectRequestsOnlyForNonOvercommittableResources, rejectResourceClaims, rejectNegativeResourceQuantities, - rejectNonPositiveHostMemoryCapacity, rejectInvalidResourceNames, rejectFractionalExtendedResources, rejectMisalignedHugepageQuantities, @@ -68,7 +64,6 @@ var DefaultValidationRules = []ValidationRule{ validateSGLangHiCache, rejectInvalidKernelCheckAnnotation, rejectUnsupportedLMCacheRole, - rejectSGLangRedisL2ScaleOut, } // SetupCacheBackendWebhookWithManager registers the defaulting and @@ -107,9 +102,7 @@ func (v *CacheBackendValidator) ValidateCreate(ctx context.Context, cb *cachev1a // itself cannot express and the controller cannot close on the operator's behalf — // never for something a validation rule could simply reject. func collectWarnings(cb *cachev1alpha1.CacheBackend) admission.Warnings { - var w admission.Warnings - w = append(w, warnMooncakeEngineHostNetwork(cb)...) - return w + return nil } // ValidateUpdate implements [admission.Validator]. Updates only reject diff --git a/internal/webhook/v1alpha1/cachebackend_validator_test.go b/internal/webhook/v1alpha1/cachebackend_validator_test.go index 97792513..bb9e653d 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator_test.go +++ b/internal/webhook/v1alpha1/cachebackend_validator_test.go @@ -6,257 +6,38 @@ package v1alpha1 import ( "context" - cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" - builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" - backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" - adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" - corev1 "k8s.io/api/core/v1" + "strings" + "testing" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" - "strings" - "testing" -) -// newBackend returns a minimally-valid managed CacheBackend the test cases -// derive from. Tests deep-copy and mutate the relevant fields rather than -// re-declaring the whole spec each time. -func newBackend() *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "cb", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, - Type: cachev1alpha1.CacheBackendTypeLMCache, - }, - } -} - -func managedLMCacheServer(cb *cachev1alpha1.CacheBackend) *cachev1alpha1.LMCacheServerRemoteStorageSpec { - if cb.Spec.RemoteStorage == nil { - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{} - } - cb.Spec.RemoteStorage.Provider = cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer - cb.Spec.RemoteStorage.Ownership = cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged - if cb.Spec.RemoteStorage.LMCacheServer == nil { - cb.Spec.RemoteStorage.LMCacheServer = &cachev1alpha1.LMCacheServerRemoteStorageSpec{} - } - return cb.Spec.RemoteStorage.LMCacheServer -} + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" + "github.com/cachebox-project/inference-cache/internal/enginebinding" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" + adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" +) func i32p(v int32) *int32 { return &v } -func defaultShippingRegistry() *adapterruntime.Registry { - registry := adapterruntime.NewRegistry() - registry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{})) - registry.Register(builtinruntime.NewVLLMLMCacheAdapter(builtinruntime.SubscriberConfig{})) - registry.Register(builtinruntime.NewSGLangLMCacheAdapter(builtinruntime.SubscriberConfig{})) - registry.Register(builtinruntime.NewSGLangHiCacheAdapter(builtinruntime.SubscriberConfig{})) - return registry -} - -func shippingValidator() *CacheBackendValidator { - return &CacheBackendValidator{Registry: defaultShippingRegistry()} -} - -func newHiCacheBackend() *cachev1alpha1.CacheBackend { - return &cachev1alpha1.CacheBackend{ - ObjectMeta: metav1.ObjectMeta{Name: "hicache", Namespace: "team-a"}, - Spec: cachev1alpha1.CacheBackendSpec{ - Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, - Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, - Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app": "sglang"}, - }, - HiCache: &cachev1alpha1.SGLangHiCacheSpec{ - Ratio: "2.0", - WritePolicy: cachev1alpha1.SGLangHiCacheWriteThroughSelective, - IOBackend: cachev1alpha1.SGLangHiCacheIOKernel, - MemoryLayout: cachev1alpha1.SGLangHiCacheMemoryPageFirst, - }, - Observation: &cachev1alpha1.CacheBackendObservationSpec{ModelID: "model-a"}, - }, - } -} - -// requireInvalidWithCause runs v against cb and asserts the response is an -// aggregated Invalid status whose causes contain the substring wantMsg on -// field wantField. Centralising the assertion keeps the per-rule tests one -// line and the error-shape contract pinned in one place. -func requireInvalidWithCause(t *testing.T, v *CacheBackendValidator, cb *cachev1alpha1.CacheBackend, wantField, wantMsg string) { - t.Helper() - _, err := v.ValidateCreate(context.Background(), cb) - if err == nil { - t.Fatalf("expected validation error, got nil") - } - statusErr, ok := err.(*apierrors.StatusError) - if !ok { - t.Fatalf("expected *apierrors.StatusError, got %T: %v", err, err) - } - if !apierrors.IsInvalid(err) { - t.Fatalf("expected Invalid status, got %v", statusErr.Status()) - } - if statusErr.Status().Details == nil { - t.Fatalf("Invalid status has no details: %v", statusErr.Status()) - } - for _, c := range statusErr.Status().Details.Causes { - if c.Field == wantField && strings.Contains(c.Message, wantMsg) { - return - } - } - t.Fatalf("no cause on field %q containing %q; got causes: %+v", - wantField, wantMsg, statusErr.Status().Details.Causes) -} - -// requireUpdateInvalidWithCause is the ValidateUpdate-equivalent of -// requireInvalidWithCause. Asserts the (old, new) pair fails validation -// with an Invalid status carrying the named field + message substring. -func requireUpdateInvalidWithCause(t *testing.T, v *CacheBackendValidator, oldCB, newCB *cachev1alpha1.CacheBackend, wantField, wantMsg string) { - t.Helper() - _, err := v.ValidateUpdate(context.Background(), oldCB, newCB) - if err == nil { - t.Fatalf("expected validation error on update, got nil") - } - statusErr, ok := err.(*apierrors.StatusError) - if !ok { - t.Fatalf("expected *apierrors.StatusError, got %T: %v", err, err) - } - if !apierrors.IsInvalid(err) { - t.Fatalf("expected Invalid status, got %v", statusErr.Status()) - } - if statusErr.Status().Details == nil { - t.Fatalf("Invalid status has no details: %v", statusErr.Status()) - } - for _, c := range statusErr.Status().Details.Causes { - if c.Field == wantField && strings.Contains(c.Message, wantMsg) { - return - } - } - t.Fatalf("no cause on field %q containing %q; got causes: %+v", - wantField, wantMsg, statusErr.Status().Details.Causes) -} - -func TestValidator_HappyPath_LMCacheAdmitted(t *testing.T) { - v := shippingValidator() - cb := newBackend() - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("happy-path LMCache rejected: %v", err) - } -} - -func mooncakeBackendWithEngineHostNetwork(optIn bool) *cachev1alpha1.CacheBackend { - cb := newBackend() - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, - EngineHostNetwork: optIn, - } - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - } - return cb -} - -func setCanonicalExternalStorage(cb *cachev1alpha1.CacheBackend, endpoint string) { - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, - Endpoint: endpoint, - } -} - -func setCanonicalMooncakeStorage(cb *cachev1alpha1.CacheBackend) { - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Type = cachev1alpha1.CacheBackendTypeLMCache - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderMooncake, - Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, - } -} - -func TestValidator_WarningTextStaysConcise(t *testing.T) { - // The Kubernetes API conventions ask for warnings within a concise budget so - // clients render them reliably. A warning that gets truncated — or dropped — is - // exactly the silent failure this warning exists to prevent, so guard the budget - // here rather than trusting review to catch a future edit that pads it out. - for _, w := range []string{mooncakeEngineHostNetworkWarning} { - if got := len(w); got > maxWarningLen { - t.Fatalf("warning is %d chars, want <= %d — put the detail in the docs, not the warning:\n%q", - got, maxWarningLen, w) - } - } -} - -func sglangLMCacheBackend() *cachev1alpha1.CacheBackend { - cb := newBackend() // Type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ +func newBackend() *cachev1alpha1.CacheBackend { + backend := validPodLocalMPBackend() + backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, } - return cb -} - -func TestValidator_SameNamespaceEndpointAdmitted(t *testing.T) { - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "team-a-cache.team-a.svc.cluster.local:9000") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("same-namespace endpoint rejected: %v", err) - } + return backend } -func TestValidator_ExternalHostnamePassesThrough(t *testing.T) { - // External hostnames are not in-cluster Service DNS — the cross-namespace - // rule has no namespace to compare against and must let them through. - // Use a bare host:port (the canonical External shape; the LMCache - // adapter prepends the lm:// scheme on injection). - v := &CacheBackendValidator{Registry: stubRegistryWithExternal()} - cb := newBackend() - setCanonicalExternalStorage(cb, "cache.example.com:8200") - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeVLLM - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{} - if _, err := v.ValidateCreate(context.Background(), cb); err != nil { - t.Fatalf("external hostname rejected: %v", err) - } -} - -func TestValidator_AggregatesMultipleViolations(t *testing.T) { - // Two independent violations on a single CR must both appear in the - // rejection's status.details.causes, so kubectl prints them together. - // Here: non-positive host memory plus scale-to-zero with autoscaling and no - // explicit minReplicas. Both rules should fire on - // the same spec. +func TestValidator_HappyPath_LMCacheAdmitted(t *testing.T) { v := shippingValidator() cb := newBackend() - zeroQuantity := resource.MustParse("0") - cb.Spec.LMCache = &cachev1alpha1.LMCacheEngineSpec{ - HostMemory: &cachev1alpha1.CacheBackendHostMemorySpec{Capacity: &zeroQuantity}, - } - zero := int32(0) - cb.Spec.Replicas = &zero - cb.Spec.Autoscaling = &cachev1alpha1.CacheBackendAutoscalingSpec{MaxReplicas: 3} - - _, err := v.ValidateCreate(context.Background(), cb) - statusErr, ok := err.(*apierrors.StatusError) - if !ok { - t.Fatalf("expected *apierrors.StatusError, got %T: %v", err, err) - } - if statusErr.Status().Details == nil || len(statusErr.Status().Details.Causes) < 2 { - t.Fatalf("expected >=2 causes, got %+v", statusErr.Status().Details) + if _, err := v.ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("happy-path LMCache rejected: %v", err) } } @@ -296,121 +77,156 @@ func TestValidator_PluggableRuleAppendable(t *testing.T) { requireInvalidWithCause(t, v, cb, "spec", "synthetic") } -// stubVLLMLMCacheAdapter is a hermetic stand-in for the production -// vLLM+LMCache adapter that exercises the validator without dragging in the -// reference-stack adapter wiring. It supports exactly the vLLM/LMCache pair -// and exposes it via PairLister so the registry surfaces the same option to -// admission error messages. -type stubVLLMLMCacheAdapter struct{} - -func (stubVLLMLMCacheAdapter) Supports(rt adapterruntime.RuntimeID, cb *cachev1alpha1.CacheBackend) bool { - if cb == nil { - return false +func TestSetupCacheBackendWebhookRequiresRegistry(t *testing.T) { + if err := SetupCacheBackendWebhookWithManager(nil, nil); err == nil || !strings.Contains(err.Error(), "registry is required") { + t.Fatalf("SetupCacheBackendWebhookWithManager error = %v, want missing-registry error", err) } - return rt == adapterruntime.RuntimeVLLM && cb.Spec.Type == cachev1alpha1.CacheBackendTypeLMCache } -func (stubVLLMLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) bool { - return binding == nil || - binding.Protocol == backendadapter.ProtocolLMCache || - binding.Protocol == backendadapter.ProtocolMooncakeStore +func TestValidator_VLLMRoleReadOnlyRejected(t *testing.T) { + // vLLM renders ReadOnly as kv_consumer, but LMCache 0.5.3 does not enforce + // that directionality. Admission must reject the unsupported API promise. + v := &CacheBackendValidator{Registry: defaultShippingRegistry()} + cb := newBackend() + cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ + Role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, + } + requireInvalidWithCause(t, v, cb, "spec.integration.role", "directional cache access") } -func (stubVLLMLMCacheAdapter) ResolveCacheServer(*cachev1alpha1.CacheBackend) (*corev1.PodSpec, *corev1.Service, error) { - return nil, nil, nil -} -func (stubVLLMLMCacheAdapter) InjectEngineConfig(*corev1.PodSpec, *backendadapter.Binding, *cachev1alpha1.CacheBackend) error { - return nil -} -func (stubVLLMLMCacheAdapter) InjectRouterConfig(*corev1.PodSpec, *backendadapter.Binding, *cachev1alpha1.CacheBackend) error { - return nil -} -func (stubVLLMLMCacheAdapter) ObservationSidecar(*cachev1alpha1.CacheBackend, *corev1.Pod) (*corev1.Container, error) { - return nil, nil -} -func (stubVLLMLMCacheAdapter) SupportedPairs() []adapterruntime.SupportedPair { - return []adapterruntime.SupportedPair{{ - Runtime: adapterruntime.RuntimeVLLM, - Backend: cachev1alpha1.CacheBackendTypeLMCache, - }} -} -func (stubVLLMLMCacheAdapter) ReservedArgs() []string { - return []string{"--kv-transfer-config"} +func defaultShippingRegistry() *adapterruntime.Registry { + registry := adapterruntime.NewRegistry() + registry.Register(builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{})) + registry.Register(builtinruntime.NewSGLangLMCacheAdapter(builtinruntime.SubscriberConfig{})) + registry.Register(builtinruntime.NewSGLangHiCacheAdapter(builtinruntime.SubscriberConfig{})) + return registry } -func (stubVLLMLMCacheAdapter) ReservedEnv() []string { - return []string{"VLLM_USE_V1", "LMCACHE_REMOTE_URL", "INFERENCECACHE_FAIL_OPEN", "PYTHONHASHSEED"} + +func shippingValidator() *CacheBackendValidator { + return &CacheBackendValidator{Registry: defaultShippingRegistry()} } -func (stubVLLMLMCacheAdapter) EngineContainerName() string { return "vllm" } -// stubRegistry returns a Registry with the stub vLLM+LMCache adapter -// installed. Hermetic — tests don't depend on the in-tree -// builtin adapter composition, so they keep passing if a future adapter joins -// or leaves the default set. External ownership uses this same runtime adapter; -// it is a remote-storage binding property, not a separate cache type. -func stubRegistry() *adapterruntime.Registry { - r := adapterruntime.NewRegistry() - r.Register(builtinruntime.NewVLLMLMCacheAdapter(builtinruntime.SubscriberConfig{})) - return r +type rejectingBindingAdapter struct { + adapterruntime.KVCacheRuntimeAdapter } -// stubRegistryWithExternal uses the same LMCache runtime adapter because -// external ownership is a remote-storage property, not a cache type. -func stubRegistryWithExternal() *adapterruntime.Registry { - return stubRegistry() +func (rejectingBindingAdapter) SupportsBinding(*backendadapter.Binding) bool { return false } + +func TestValidatorTypedLMCacheChecksRuntimeBindingCapability(t *testing.T) { + cb := newBackend() + registry := adapterruntime.NewRegistry() + registry.Register(rejectingBindingAdapter{ + KVCacheRuntimeAdapter: builtinruntime.NewVLLMLMCacheMPAdapter(builtinruntime.SubscriberConfig{}), + }) + + requireInvalidWithCause(t, &CacheBackendValidator{Registry: registry}, cb, + "spec.remoteStorage.provider", "does not accept remote-storage protocol") } -func TestSetupCacheBackendWebhookRequiresRegistry(t *testing.T) { - if err := SetupCacheBackendWebhookWithManager(nil, nil); err == nil || !strings.Contains(err.Error(), "registry is required") { - t.Fatalf("SetupCacheBackendWebhookWithManager error = %v, want missing-registry error", err) +func newHiCacheBackend() *cachev1alpha1.CacheBackend { + return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "hicache", Namespace: "team-a"}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeSGLang, + Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, + HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ + Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, + Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, + }, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "sglang"}}, + }, } } -// withVLLMOverrides returns a fresh stub-LMCache backend whose integration -// declares vLLM and carries the supplied EngineInjectionOverrides — the -// admission-test backing for the engine-overrides rule. -func withVLLMOverrides(o cachev1alpha1.EngineInjectionOverrides) *cachev1alpha1.CacheBackend { - cb := newBackend() - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - EngineOverrides: &o, +func requireInvalidWithCause(t *testing.T, validator *CacheBackendValidator, cb *cachev1alpha1.CacheBackend, wantField, wantMsg string) { + t.Helper() + _, err := validator.ValidateCreate(context.Background(), cb) + if err == nil || !apierrors.IsInvalid(err) { + t.Fatalf("ValidateCreate() error = %v, want Invalid", err) + } + statusErr := err.(*apierrors.StatusError) + for _, cause := range statusErr.Status().Details.Causes { + if cause.Field == wantField && strings.Contains(cause.Message, wantMsg) { + return + } } - return cb + t.Fatalf("no cause on field %q containing %q: %+v", wantField, wantMsg, statusErr.Status().Details.Causes) } -// withSGLangOverrides builds a (sglang, LMCache) CacheBackend carrying the -// given engineOverrides. The SGLang override tests use the real -// defaultShippingRegistry (not stubRegistry) so the SGLang adapter is the one -// the reserved-args/env check consults — proving the check is keyed on the -// SELECTED adapter, not a hardcoded vLLM list. -func withSGLangOverrides(o cachev1alpha1.EngineInjectionOverrides) *cachev1alpha1.CacheBackend { - cb := newBackend() // type=LMCache - cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - EngineOverrides: &o, +func requireUpdateInvalidWithCause(t *testing.T, validator *CacheBackendValidator, oldCB, newCB *cachev1alpha1.CacheBackend, wantField, wantMsg string) { + t.Helper() + _, err := validator.ValidateUpdate(context.Background(), oldCB, newCB) + if err == nil || !apierrors.IsInvalid(err) { + t.Fatalf("ValidateUpdate() error = %v, want Invalid", err) } - return cb + statusErr := err.(*apierrors.StatusError) + for _, cause := range statusErr.Status().Details.Causes { + if cause.Field == wantField && strings.Contains(cause.Message, wantMsg) { + return + } + } + t.Fatalf("no cause on field %q containing %q: %+v", wantField, wantMsg, statusErr.Status().Details.Causes) } -func TestValidator_VLLMRoleReadOnlyRejected(t *testing.T) { - // vLLM renders ReadOnly as kv_consumer, but LMCache 0.5.3 does not enforce - // that directionality. Admission must reject the unsupported API promise. - v := &CacheBackendValidator{Registry: defaultShippingRegistry()} - cb := newBackend() - cb.Spec.Integration = &cachev1alpha1.CacheBackendIntegrationSpec{ - Role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, +func TestValidatorAdmitsTypedPodLocal(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.PodLocal.Server.L1Capacity = resource.MustParse("4Gi") + if _, err := shippingValidator().ValidateCreate(context.Background(), cb); err != nil { + t.Fatalf("typed PodLocal rejected: %v", err) } - requireInvalidWithCause(t, v, cb, "spec.integration.role", "directional cache access") } -// eventsOnlyIntegration returns an integration spec wired for the events-only -// (tier-1 routing) mode. Centralised so the events-only tests set the mode the -// one way the implementation reads it (via spec.integration.mode). -func eventsOnlyIntegration() *cachev1alpha1.CacheBackendIntegrationSpec { - return &cachev1alpha1.CacheBackendIntegrationSpec{ - Mode: cachev1alpha1.CacheBackendIntegrationModeEventsOnly, - } +func TestValidatorRejectsTopologyLessLMCache(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.Topology = "" + requireInvalidWithCause(t, shippingValidator(), cb, "spec.lmCache.topology", "required") +} + +func TestValidatorSGLangHiCacheCurrentContract(t *testing.T) { + t.Run("accepted", func(t *testing.T) { + if _, err := shippingValidator().ValidateCreate(context.Background(), newHiCacheBackend()); err != nil { + t.Fatalf("valid SGLangHiCache rejected: %v", err) + } + }) + + t.Run("remote storage rejected", func(t *testing.T) { + cb := newHiCacheBackend() + cb.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipManaged, + Redis: &cachev1alpha1.RedisRemoteStorageSpec{}, + } + requireInvalidWithCause(t, shippingValidator(), cb, "spec.remoteStorage.provider", "does not accept") + }) + + t.Run("hiCache block rejected for LMCache", func(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.HiCache = &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"} + requireInvalidWithCause(t, shippingValidator(), cb, "spec.hiCache", "only valid") + }) + + t.Run("invalid ratio rejected", func(t *testing.T) { + cb := newHiCacheBackend() + cb.Spec.HiCache.Ratio = "NaN" + requireInvalidWithCause(t, shippingValidator(), cb, "spec.hiCache.ratio", "finite number") + }) } -// Sanity check on the package-level wiring: SetupCacheBackendWebhookWithManager -// is exercised by manager start-up; the runtime.Object interface is the only -// thing we can sanity-check here without a controller manager. -var _ runtime.Object = (*cachev1alpha1.CacheBackend)(nil) +func TestValidatorRejectsInvalidKernelCheckAnnotation(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Annotations = map[string]string{enginebinding.AnnotationLMCacheKernelCheck: "strcit"} + requireInvalidWithCause(t, shippingValidator(), cb, + "metadata.annotations[inferencecache.io/lmcache-kernel-check]", "must be one of") +} + +func TestValidatorRejectsEventsOnlyRemoteStorage(t *testing.T) { + cb := newBackend() + cb.Spec.Integration.Mode = cachev1alpha1.CacheBackendIntegrationModeEventsOnly + requireInvalidWithCause(t, shippingValidator(), cb, "spec.remoteStorage", "remove spec.remoteStorage") +} + +func TestValidatorRejectsUnknownRuntime(t *testing.T) { + cb := validPodLocalMPBackend() + cb.Spec.Runtime = cachev1alpha1.CacheBackendRuntime("UnknownRuntime") + requireInvalidWithCause(t, shippingValidator(), cb, "spec.runtime", "no runtime adapter supports") +} diff --git a/pkg/adapters/backend/backend.go b/pkg/adapters/backend/backend.go index 61d3f923..0a9ff887 100644 --- a/pkg/adapters/backend/backend.go +++ b/pkg/adapters/backend/backend.go @@ -20,9 +20,7 @@ import ( type Protocol string const ( - ProtocolLMCache Protocol = "lm" - ProtocolRESP Protocol = "resp" - ProtocolMooncakeStore Protocol = "mooncakestore" + ProtocolRESP Protocol = "resp" ) // Binding is the structured connection information an engine adapter accepts. @@ -121,10 +119,6 @@ func ProtocolFor(storage *cachev1alpha1.CacheBackendRemoteStorageSpec) (Protocol switch storage.Provider { case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: return ProtocolRESP, nil - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - return ProtocolLMCache, nil - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - return ProtocolMooncakeStore, nil default: return "", fmt.Errorf("%w: unknown provider=%q", ErrNoProvider, storage.Provider) } diff --git a/pkg/adapters/backend/backend_test.go b/pkg/adapters/backend/backend_test.go index 9e8c91b6..032c1560 100644 --- a/pkg/adapters/backend/backend_test.go +++ b/pkg/adapters/backend/backend_test.go @@ -13,12 +13,12 @@ import ( func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ - Provider: cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer, + Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, Ownership: cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal, Endpoint: " cache.example:65432 ", } - got := BindingFor(storage, ProtocolLMCache, "cache.example:65432") + got := BindingFor(storage, ProtocolRESP, "cache.example:65432") if got == nil { t.Fatal("BindingFor returned nil") } diff --git a/pkg/adapters/backend/endpoint.go b/pkg/adapters/backend/endpoint.go index 115d2f75..4611f5b0 100644 --- a/pkg/adapters/backend/endpoint.go +++ b/pkg/adapters/backend/endpoint.go @@ -6,6 +6,7 @@ package backend import ( "fmt" + "net" "strconv" "strings" "unicode" @@ -13,33 +14,25 @@ import ( cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" ) -// ValidateLMCacheEndpoint validates a bare host:port or lm://host:port. The -// port must be a decimal integer in the TCP range 1-65535. -func ValidateLMCacheEndpoint(value string) error { - raw := strings.TrimSpace(value) - if raw == "" { - return fmt.Errorf("endpoint is empty") - } - if strings.ContainsFunc(raw, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) { - return fmt.Errorf("endpoint must not contain whitespace or control characters within the host or port; use host:port or lm://host:port with no embedded spaces") +// ValidateExternalEndpoint validates the bare host:port endpoint accepted by +// the selected remote-storage provider. +func ValidateExternalEndpoint(provider cachev1alpha1.CacheBackendRemoteStorageProvider, endpoint string) error { + if provider != cachev1alpha1.CacheBackendRemoteStorageProviderRedis { + return fmt.Errorf("remote-storage provider %q has no endpoint protocol", provider) } - rest := raw - if i := strings.Index(raw, "://"); i >= 0 { - scheme := strings.ToLower(raw[:i]) - rest = raw[i+3:] - if scheme != "lm" { - return fmt.Errorf("endpoint scheme %q is not supported; use a bare host:port (the LMCache adapter adds the lm:// scheme) or an explicit lm://host:port URL", scheme) - } + value := strings.TrimSpace(endpoint) + if value == "" { + return fmt.Errorf("endpoint is empty") } - if strings.ContainsAny(rest, "/?#") { - return fmt.Errorf("endpoint must be host:port (optionally prefixed lm://); paths/queries/fragments are not part of the LMCache wire and would be silently dropped") + if strings.Contains(value, "://") { + return fmt.Errorf("endpoint schemes are not supported for remoteStorage.provider=%s; use bare host:port", provider) } - host, port, ok := splitLMCacheHostPort(rest) - if !ok || host == "" || port == "" { - return fmt.Errorf("endpoint must be a non-empty host AND port (e.g. cache.example.com:8200 or lm://cache.example.com:8200); a scheme alone, a host with no port, an empty port, or a port with no host is not a valid LMCache endpoint") + if strings.ContainsFunc(value, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) { + return fmt.Errorf("endpoint must not contain whitespace or control characters") } - if strings.IndexFunc(port, func(r rune) bool { return r < '0' || r > '9' }) >= 0 { - return fmt.Errorf("endpoint port %q must be an integer in 1-65535", port) + host, port, err := net.SplitHostPort(value) + if err != nil || host == "" || port == "" { + return fmt.Errorf("endpoint must be a non-empty host and port (for example redis.example.com:6379)") } n, err := strconv.ParseUint(port, 10, 16) if err != nil || n == 0 { @@ -47,59 +40,3 @@ func ValidateLMCacheEndpoint(value string) error { } return nil } - -func splitLMCacheHostPort(value string) (host, port string, hasPort bool) { - if value == "" { - return "", "", false - } - if strings.HasPrefix(value, "[") { - end := strings.Index(value, "]") - if end <= 1 { - return "", "", false - } - host = value[1:end] - tail := value[end+1:] - if tail == "" { - return host, "", false - } - if !strings.HasPrefix(tail, ":") || strings.Contains(tail[1:], ":") { - return "", "", false - } - return host, tail[1:], true - } - if strings.Count(value, ":") > 1 { - return "", "", false - } - if i := strings.LastIndex(value, ":"); i >= 0 { - return value[:i], value[i+1:], true - } - return value, "", false -} - -// ValidateExternalEndpoint validates an endpoint for the selected remote -// storage provider's engine-side protocol. -func ValidateExternalEndpoint(provider cachev1alpha1.CacheBackendRemoteStorageProvider, endpoint string) error { - trimmed := strings.TrimSpace(endpoint) - switch provider { - case cachev1alpha1.CacheBackendRemoteStorageProviderLMCacheServer: - return ValidateLMCacheEndpoint(trimmed) - case cachev1alpha1.CacheBackendRemoteStorageProviderRedis: - if scheme, _, ok := strings.Cut(trimmed, "://"); ok { - return fmt.Errorf("scheme %q is not supported for remoteStorage.provider=%s; use bare host:port", scheme, provider) - } - return ValidateLMCacheEndpoint(trimmed) - case cachev1alpha1.CacheBackendRemoteStorageProviderMooncake: - if scheme, address, ok := strings.Cut(trimmed, "://"); ok { - if !strings.EqualFold(scheme, "mooncakestore") { - return fmt.Errorf("scheme %q is not supported for remoteStorage.provider=%s; use bare host:port or mooncakestore://host:port", scheme, provider) - } - if strings.Contains(address, "://") { - return fmt.Errorf("nested endpoint schemes are not supported for remoteStorage.provider=%s; use mooncakestore://host:port", provider) - } - trimmed = address - } - return ValidateLMCacheEndpoint(trimmed) - default: - return fmt.Errorf("remote-storage provider %q has no endpoint protocol", provider) - } -} diff --git a/pkg/adapters/runtime/adapter.go b/pkg/adapters/runtime/adapter.go index 1dd6b87e..65f69d74 100644 --- a/pkg/adapters/runtime/adapter.go +++ b/pkg/adapters/runtime/adapter.go @@ -111,11 +111,9 @@ type KVCacheRuntimeAdapter interface { EngineContainerName() string } -// LMCacheMPRuntimeAdapter is the Phase-1 gate for adapters that understand the -// final typed LMCache topology. Legacy adapters intentionally do not implement -// it: the Pod webhook then admits a new MP Pod unmodified instead of silently -// applying the legacy in-process/flat-field wire. Phases 2-4 implement this -// interface as the shared renderer and runtime-specific MP adapters land. +// LMCacheMPRuntimeAdapter identifies adapters that understand the typed LMCache +// MP topology. The Pod webhook uses the additional validation hook before +// mutating a selected engine Pod. type LMCacheMPRuntimeAdapter interface { KVCacheRuntimeAdapter diff --git a/pkg/adapters/runtime/adapter_test.go b/pkg/adapters/runtime/adapter_test.go index 065e525b..cd39df57 100644 --- a/pkg/adapters/runtime/adapter_test.go +++ b/pkg/adapters/runtime/adapter_test.go @@ -69,7 +69,7 @@ func newCacheBackend(t cachev1alpha1.CacheBackendType, engine string) *cachev1al } func referenceBinding(endpoint string) *backendadapter.Binding { - return &backendadapter.Binding{Protocol: backendadapter.ProtocolLMCache, Endpoint: endpoint} + return &backendadapter.Binding{Protocol: backendadapter.ProtocolRESP, Endpoint: endpoint} } func TestRegistrySelectFirstMatchWins(t *testing.T) { diff --git a/site/content/en/docs/concepts/_index.md b/site/content/en/docs/concepts/_index.md index 8e8a9c48..03485d2c 100644 --- a/site/content/en/docs/concepts/_index.md +++ b/site/content/en/docs/concepts/_index.md @@ -26,9 +26,9 @@ Five are namespaced; `CacheIndex` is the only cluster-scoped type. Controllers e ### [CacheBackend]({{< relref "/docs/concepts/cachebackend/" >}}) -The primary resource an operator writes. It binds to inference-engine pods by label, -provisions a managed cache-server workload, and makes the engine's KV cache reusable across -requests. Short name `cb`. +The primary resource an operator writes. It binds to inference-engine pods by +label, injects typed engine-side cache wiring, and optionally provisions managed +Redis as a remote tier. Short name `cb`. ### [CachePolicy]({{< relref "/docs/concepts/cachepolicy/" >}}) diff --git a/site/content/en/docs/concepts/architecture.md b/site/content/en/docs/concepts/architecture.md index cccadfa2..4f2f929c 100644 --- a/site/content/en/docs/concepts/architecture.md +++ b/site/content/en/docs/concepts/architecture.md @@ -18,8 +18,9 @@ the cache-state index. The controller-runtime manager. It: -- **Reconciles the CRDs** — provisions the managed cache-server workload and Service for a - `CacheBackend`, computes readiness, and writes status. +- **Reconciles the CRDs** — provisions an explicitly selected managed Redis + provider for a `CacheBackend`, computes connector and provider readiness, and + writes status. Host-only MP and engine-local modes create no provider workload. - **Serves six CR admission webhook entries** (over TLS, via cert-manager): defaulting + validation for `CacheBackend`, `CachePolicy`, and `CacheTenant`. - **Serves the seventh entry, a mutating Pod webhook** — the *linker*. When an engine pod diff --git a/site/content/en/docs/concepts/cachebackend.md b/site/content/en/docs/concepts/cachebackend.md index 194d4047..c939e9d7 100644 --- a/site/content/en/docs/concepts/cachebackend.md +++ b/site/content/en/docs/concepts/cachebackend.md @@ -67,9 +67,14 @@ PodLocal native sidecars require Kubernetes 1.29 or newer. | `SGLangHiCache` | SGLang's native engine-local host cache; no remote binding. | `remoteStorage` is optional L3 only. Redis may be `Managed` or `External`. -Legacy topology-less `LMCacheServer` and engine-side Mooncake shapes remain in -the alpha schema only for compatibility until migration Phase 7; they are not -current production profiles and are not automatically mapped to Redis. +The removed topology-less IP providers are not accepted or automatically +mapped to Redis because that would change sharing semantics. + +For `Managed` Redis, `remoteStorage.workload` controls provider Pod scheduling +and security. The built-in managed Redis is a standalone singleton; a future +managed Redis Cluster requires provider-specific shard and replica semantics, +not a generic Deployment replica count. Mooncake is likewise planned as a new +typed MP L2 adapter rather than a restoration of its former IP wire. ## Engine integration @@ -93,9 +98,9 @@ Typed MP exposes connector and remote-storage health separately: - `RemoteStorageReady` is present only when a Redis L3 is configured; and - `Ready` composes the implemented readiness and observation gates. -`status.endpoint` is empty for host-only PodLocal MP and contains only a remote -L3 endpoint. It never publishes the loopback connector address. Other useful -fields include `matchedEnginePods`, `connector`, `remoteStorage`, +`status.remoteStorage.endpoint` exists only when a remote L3 is configured. It +never publishes the loopback connector address. Other useful fields include +`matchedEnginePods`, `connector`, `remoteStorage`, `indexParticipation`, `conditions`, and `observedGeneration`. ## Related pages diff --git a/site/content/en/docs/reference/crd-api.md b/site/content/en/docs/reference/crd-api.md index 92c687dc..54a18d53 100644 --- a/site/content/en/docs/reference/crd-api.md +++ b/site/content/en/docs/reference/crd-api.md @@ -33,22 +33,18 @@ contract follows its own compatibility policy for external consumers. | `type` | `LMCache`, `SGLangHiCache` | `LMCache` | Engine-side cache implementation. | | `lmCache` | object | — | Typed LMCache MP topology and server configuration. Current offload uses `topology: PodLocal`. | | `lmCache.podLocal.server.resources` | ResourceRequirements | required | Resources for the injected MP server; memory covers L1 plus 1Gi and CPU request is positive. | -| `remoteStorage` | object | — | Optional Redis L3 with `Managed` or `External` ownership. Legacy providers remain in the alpha schema only until Phase 7. | +| `remoteStorage` | object | — | Optional Redis L3 with `Managed` or `External` ownership. | +| `remoteStorage.workload` | object | — | Scheduling and Pod security for a managed provider; rejected for External and has no generic replica count. | | `observation` | object | — | Model identity and first-event timeout. | -| `deploymentKind` | `Deployment`, `StatefulSet` | `Deployment` | `StatefulSet` reserved/no-op. | -| `replicas` | int32 | `1` | Min 0. | -| `autoscaling` | object | — | `minReplicas`, `maxReplicas` (required), `targetCPUUtilizationPercent` (default 80). | | `integration.mode` | `Offload`, `EventsOnly` | `Offload` | Events-only = routing only. | | `integration.role` | `ReadOnly`, `WriteOnly`, `ReadWrite` | `ReadWrite` | LMCache currently admits only `ReadWrite`; directional semantics are future work. | | `integration.failOpen` | bool | `true` | `false` fails closed. | | `integration.engineOverrides` | object | — | `args` / `suppressArgs` / `env` / `suppressEnv`. | -| `integration.engineHostNetwork` | bool | `false` | Legacy engine-side Mooncake compatibility field; not used by typed MP. | | `engineSelector.matchLabels` | map | — | Equality selector over engine pod labels. | -| `template` | object | — | Narrow pod-level overrides (no containers). | | `remoteStorage..resources` | ResourceRequirements | renderer default: `requests.memory 4Gi` / `limits.memory 8Gi` | Resources for the selected managed provider container. | | `allowCrossNamespace` | bool | `false` | Opt-in cross-namespace endpoints. | -**Key `status` fields:** `endpoint`, `matchedEnginePods` (`*int32`), +**Key `status` fields:** `connector`, `remoteStorage`, `matchedEnginePods` (`*int32`), `firstKVEventObservedAt` (`*Time`), `indexParticipation` (`prefixCount`, `lastEventAt`, `hitRate *string`, `t2HitRate *string`), `failOpen`, `observedGeneration`, `conditions`. diff --git a/site/content/en/docs/reference/metrics.md b/site/content/en/docs/reference/metrics.md index bacc9d97..9e70d535 100644 --- a/site/content/en/docs/reference/metrics.md +++ b/site/content/en/docs/reference/metrics.md @@ -58,7 +58,6 @@ alerts have no series to evaluate. |---|---|---| | `inferencecache_backend_probe_result_total` | `backend`, `stage` (`ingest`/`routing`/`t2`), `result` (`ok`/`failed`/`skipped`) | Functional-probe stage results — three increments per successful call. | | `inferencecache_backend_t2_query_tokens_total` | `backend` | Monotonic tier-2 activity signal (positive deltas only). | -| `inferencecache_backend_server_restart_cascades_total` | `namespace`, `backend`, `reason` (`server_instance_changed`) | Cache-server restart cascades, rate-limited (~30s). | ## Endpoints From 690812158a7dbc5623f602bb30c4f25aa2a271d0 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 12 Aug 2026 16:30:46 -0700 Subject: [PATCH 08/13] Add NodeLocal shared LMCache MP servers Follow engine scheduling with one same-node server Pod per active node, add typed lifecycle and coverage status, enforce canonical cache domains, and inject runtime-specific NodeLocal connectors. Use UID-scoped shared-memory identities with startup/status verification, retain warm servers during idle windows, and document the completed local and GPU validation. Signed-off-by: Yue Sun --- .github/workflows/phase5-upgrade-smoke.yml | 50 - api/v1alpha1/cachebackend_types.go | 138 +- api/v1alpha1/cachebackend_types_test.go | 35 +- api/v1alpha1/zz_generated.deepcopy.go | 50 +- .../inferencecache.io_cachebackends.yaml | 1297 +++++------------ config/observability/lmcache-podmonitor.yaml | 9 +- config/rbac/role.yaml | 3 + config/samples/README.md | 22 +- .../samples/cache_v1alpha1_cachebackend.yaml | 2 +- config/samples/cachebackend-cpu-override.yaml | 2 +- config/samples/cachebackend-events-only.yaml | 2 +- config/samples/cachebackend-external.yaml | 2 +- config/samples/cachebackend-lmcache.yaml | 2 +- .../samples/cachebackend-sglang-hicache.yaml | 2 +- .../cachebackend-sglang-host-only.yaml | 2 +- ...chebackend-sglang-nodelocal-host-only.yaml | 44 + ...ackend-sglang-podlocal-external-redis.yaml | 2 +- ...achebackend-sglang-podlocal-host-only.yaml | 2 +- ...backend-sglang-podlocal-managed-redis.yaml | 2 +- config/samples/cachebackend-sglang.yaml | 6 +- ...cachebackend-vllm-nodelocal-host-only.yaml | 44 + ...ebackend-vllm-podlocal-external-redis.yaml | 2 +- .../cachebackend-vllm-podlocal-host-only.yaml | 2 +- ...hebackend-vllm-podlocal-managed-redis.yaml | 2 +- config/samples/cachebackend-with-engine.yaml | 8 +- .../samples/cachebackend-with-override.yaml | 6 +- config/samples/recipe-cpu-dev.yaml | 8 +- config/samples/recipe-external-cache.yaml | 3 +- config/samples/recipe-gpu-production.yaml | 6 +- config/samples/recipe-multi-tenant.yaml | 6 +- config/samples/recipe-tuning.yaml | 3 +- docs/cli/doctor.md | 1 + docs/concepts/cachebackend-engine-binding.md | 67 +- docs/design/cachebackend-api.md | 109 +- .../lmcache-multiprocess-migration-roadmap.md | 680 ++++++--- docs/design/lmcache-server-persistence.md | 11 +- docs/design/sglang-lmcache-mp-mode.md | 9 +- docs/quickstart.md | 32 +- .../reference-stack/manifests/deployment.yaml | 10 +- .../manifests/sglang-lmcache/deployment.yaml | 3 +- .../scripts/default_install_smoke.sh | 158 +- .../scripts/phase5_upgrade_smoke.sh | 206 --- .../builtin/runtime/lmcache_mp_nodelocal.go | 535 +++++++ .../runtime/lmcache_mp_nodelocal_test.go | 464 ++++++ .../builtin/runtime/lmcache_mp_renderer.go | 2 +- .../builtin/runtime/sglang_lmcache.go | 74 +- .../builtin/runtime/vllm_lmcache_mp.go | 84 +- internal/cli/doctor/checks/checks_test.go | 30 + internal/cli/doctor/checks/podaudit.go | 41 +- internal/cli/doctor/finding.go | 5 + internal/controller/cachebackend_dispatch.go | 17 +- .../cachebackend_lmcache_mp_status.go | 268 +++- .../cachebackend_lmcache_mp_status_test.go | 258 ++++ .../cachebackend_lmcache_nodelocal.go | 251 ++++ .../cachebackend_mp_lifecycle_test.go | 334 +++++ ...cachebackend_nodelocal_integration_test.go | 104 ++ .../controller/cachebackend_reconciler.go | 59 +- .../cachebackend_reconciler_test.go | 42 + .../controller/cachebackend_serverless.go | 4 +- internal/controller/cachebackend_status.go | 2 +- internal/controller/cachebackend_workload.go | 1 + internal/controller/cacheindex_controller.go | 31 +- .../controller/cacheindex_controller_test.go | 14 +- .../contract_coverage_sweep_test.go | 149 +- internal/enginebinding/metadata.go | 34 + .../webhook/pod/envtest_integration_test.go | 67 + internal/webhook/pod/podinjector.go | 75 +- internal/webhook/pod/podinjector_test.go | 116 +- .../cachebackend_defaulter_envtest_test.go | 75 +- .../cachebackend_lmcache_mp_validation.go | 62 +- ...cachebackend_lmcache_mp_validation_test.go | 75 +- .../v1alpha1/cachebackend_validator.go | 104 +- .../v1alpha1/cachebackend_validator_test.go | 146 +- internal/webhook/v1alpha1/doc.go | 3 +- site/content/en/docs/reference/cli-doctor.md | 2 +- 75 files changed, 4609 insertions(+), 1969 deletions(-) delete mode 100644 .github/workflows/phase5-upgrade-smoke.yml create mode 100644 config/samples/cachebackend-sglang-nodelocal-host-only.yaml create mode 100644 config/samples/cachebackend-vllm-nodelocal-host-only.yaml delete mode 100755 docs/reference-stack/scripts/phase5_upgrade_smoke.sh create mode 100644 internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go create mode 100644 internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go create mode 100644 internal/controller/cachebackend_lmcache_nodelocal.go create mode 100644 internal/controller/cachebackend_nodelocal_integration_test.go diff --git a/.github/workflows/phase5-upgrade-smoke.yml b/.github/workflows/phase5-upgrade-smoke.yml deleted file mode 100644 index aa8af5a6..00000000 --- a/.github/workflows/phase5-upgrade-smoke.yml +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -name: phase5-upgrade-smoke - -on: - pull_request: - branches: [main] - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - typed-object-upgrade: - runs-on: ubuntu-latest - timeout-minutes: 25 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - - - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 - with: - go-version-file: go.mod - - - name: Install kind - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1 - with: - install_only: true - - - name: Upgrade Phase 5 typed objects to Phase 7 - env: - TAG: ${{ github.sha }} - run: docs/reference-stack/scripts/phase5_upgrade_smoke.sh - - - name: Upload upgrade-smoke logs on failure - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: phase5-upgrade-smoke-logs - path: /tmp/phase5-upgrade-smoke-logs/ - if-no-files-found: ignore diff --git a/api/v1alpha1/cachebackend_types.go b/api/v1alpha1/cachebackend_types.go index ed336113..2acb87d6 100644 --- a/api/v1alpha1/cachebackend_types.go +++ b/api/v1alpha1/cachebackend_types.go @@ -20,6 +20,12 @@ type CacheBackendRuntime string const ( CacheBackendRuntimeVLLM CacheBackendRuntime = "VLLM" CacheBackendRuntimeSGLang CacheBackendRuntime = "SGLang" + + // CacheBackendDomainLabel is the sole supported engineSelector key. Its + // namespace-scoped value identifies one runtime/model/KV-layout/trust + // compatibility domain. Engine Pods may carry other labels, but cache + // ownership is intentionally independent of them. + CacheBackendDomainLabel = "inferencecache.io/cache-domain" ) // +kubebuilder:validation:Enum=LMCache;SGLangHiCache @@ -208,20 +214,32 @@ type LMCachePodLocalSpec struct { Server *LMCachePodLocalServerSpec `json:"server"` } -// LMCacheNodeLocalServerSpec describes the future one-server-per-node MP -// topology. The shape is published now so the API does not need another -// topology redesign, but admission rejects NodeLocal until Phase 8. +// LMCacheNodeLocalServerSpec configures the controller-owned LMCache MP server +// shared by selected engine Pods on one node chosen by the inference system. type LMCacheNodeLocalServerSpec struct { + // Image is the digest-pinned LMCache server image. The same image is used + // by the lightweight engine startup gate; CacheBackend never changes the + // inference-engine image. Image string `json:"image"` + // Port is the node-bound LMCache MP data port. // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=65535 Port int32 `json:"port"` + // HTTPPort is the node-bound FastAPI health/control port used by probes and + // by the engine startup gate to verify the same-node server identity. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + HTTPPort int32 `json:"httpPort"` + + // L1Capacity is one shared host-memory budget per active engine node, not per + // selected engine Pod. // +kubebuilder:validation:XValidation:rule="quantity(string(self)).isGreaterThan(quantity('0'))",message="l1Capacity must be greater than zero" L1Capacity resource.Quantity `json:"l1Capacity"` - // MaxGPUWorkers bounds workers serving GPU-backed engine clients. + // MaxGPUWorkers bounds workers serving GPU-backed engine clients and must + // cover the maximum number of engine instances expected on one node. // +kubebuilder:validation:Minimum=1 MaxGPUWorkers int32 `json:"maxGPUWorkers"` @@ -229,27 +247,65 @@ type LMCacheNodeLocalServerSpec struct { // +kubebuilder:validation:Minimum=1 MaxCPUWorkers int32 `json:"maxCPUWorkers"` + // Resources are applied to every per-node server Pod. Admission requires + // memory request and limit headroom above the shared L1 budget. Resources corev1.ResourceRequirements `json:"resources"` } -// LMCacheNodeLocalSchedulingSpec configures placement of the future per-node -// MP server workload. +// LMCacheNodeLocalSchedulingSpec configures server-Pod scheduling details that +// are independent of node placement. The controller derives the exact node +// from already-scheduled selected engine Pods; these fields never constrain or +// rewrite inference-engine placement. type LMCacheNodeLocalSchedulingSpec struct { + // Tolerations are merged with the tolerations of an engine already running + // on the target node. They allow the server Pod to pass that node's taints + // without selecting a different node. // +optional - NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` // +optional - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` // +optional - Affinity *corev1.Affinity `json:"affinity,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // +optional + SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` + + // +optional + PriorityClassName string `json:"priorityClassName,omitempty"` + + // +optional + SchedulerName string `json:"schedulerName,omitempty"` + + // RuntimeClassName overrides the runtime inherited from the engine Pod used + // to place this server. Clusters that do not make the NVIDIA runtime the + // default can use this field to provide GPU visibility without reserving + // allocatable GPUs. + // +optional + RuntimeClassName *string `json:"runtimeClassName,omitempty"` + + // +optional + // +kubebuilder:validation:Minimum=0 + TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"` } -// LMCacheNodeLocalSpec configures the future one-server-per-eligible-node -// topology. It is rejected at admission until its controller exists. +// LMCacheNodeLocalSpec configures one shared MP server per node that currently +// hosts at least one selected engine Pod. type LMCacheNodeLocalSpec struct { Server *LMCacheNodeLocalServerSpec `json:"server"` + // IdleRetentionSeconds keeps an otherwise healthy per-node server alive + // after the final selected engine leaves that node. A new matching engine + // scheduled there during the window reuses the same server Pod and shared + // L1. Zero requests immediate deletion. + // +kubebuilder:default=300 + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=86400 + IdleRetentionSeconds int32 `json:"idleRetentionSeconds"` + + // Scheduling optionally overrides server-Pod runtime and operational fields. + // It cannot select nodes; inference-engine scheduling remains authoritative. // +optional Scheduling *LMCacheNodeLocalSchedulingSpec `json:"scheduling,omitempty"` } @@ -258,16 +314,15 @@ type LMCacheNodeLocalSpec struct { // fields apply to the connector or node-local MP worker, not to a remote // storage provider. type LMCacheEngineSpec struct { - // Topology selects the canonical LMCache MP server placement. PodLocal is - // implemented first; NodeLocal is reserved and rejected until Phase 8. + // Topology selects the canonical LMCache MP server placement. Topology LMCacheTopology `json:"topology"` // PodLocal configures one MP server in each selected engine Pod. // +optional PodLocal *LMCachePodLocalSpec `json:"podLocal,omitempty"` - // NodeLocal configures a future per-node MP server. Admission currently - // rejects this block so it can never be accepted as inert configuration. + // NodeLocal configures a controller-owned, engine-demand-driven per-node MP + // server pool. // +optional NodeLocal *LMCacheNodeLocalSpec `json:"nodeLocal,omitempty"` @@ -451,11 +506,16 @@ type CacheBackendSpec struct { // +optional Integration *CacheBackendIntegrationSpec `json:"integration,omitempty"` - // EngineSelector selects which engine pods this CacheBackend claims via - // equality-based label matching over the pod's labels: every key/value - // in MatchLabels must be present on the pod. The full - // metav1.LabelSelector surface (matchExpressions, operator-based - // selection) is NOT exposed today — only MatchLabels. + // EngineSelector selects which engine pods this CacheBackend claims. A + // non-empty selector must contain exactly one MatchLabels entry whose key is + // inferencecache.io/cache-domain. The value is an operator-chosen, + // namespace-scoped compatibility-domain ID; the selected Pods must carry the + // same label. Pods may carry other labels, but they do not participate in + // CacheBackend ownership. The full metav1.LabelSelector surface + // (matchExpressions, operator-based selection) is NOT exposed today. + // Admission requires the domain value to be unique among CacheBackends in + // the same namespace. Every engine Pod must have exactly one CacheBackend + // owner. // Pods that match get runtime-adapter engine wiring injected by the // mutating Pod admission webhook at pod CREATE time. LMCache MP adapters // inject the local connector immediately; when an optional managed Redis @@ -728,7 +788,7 @@ type CacheBackendConnectorStatus struct { ReadyEnginePods int32 `json:"readyEnginePods,omitempty"` // DesiredServers is one per selected engine Pod for PodLocal and one per - // eligible node for NodeLocal. + // distinct active scheduled engine node for NodeLocal. // +kubebuilder:validation:Minimum=0 DesiredServers int32 `json:"desiredServers,omitempty"` @@ -745,6 +805,33 @@ type CacheBackendConnectorStatus struct { // reachable MP server. // +kubebuilder:validation:Minimum=0 UncoveredEnginePods int32 `json:"uncoveredEnginePods,omitempty"` + + // EnginePodCoverage reports the connector verdict for every active selected + // engine Pod. The list is keyed by Pod name and sorted by name by the + // controller so operators can identify the exact uncovered instance. + // +optional + // +listType=map + // +listMapKey=name + EnginePodCoverage []CacheBackendEnginePodCoverageStatus `json:"enginePodCoverage,omitempty"` +} + +// CacheBackendEnginePodCoverageStatus is the per-engine connector verdict. +// Ready means the engine Pod itself is Ready; Covered means the current +// CacheBackend generation has exactly one healthy reachable MP server for it. +type CacheBackendEnginePodCoverageStatus struct { + // Name is the selected engine Pod name in the CacheBackend namespace. + Name string `json:"name"` + + // NodeName is the scheduled node. It is empty while the Pod is pending. + // +optional + NodeName string `json:"nodeName,omitempty"` + + Ready bool `json:"ready"` + + Covered bool `json:"covered"` + + // Reason is a stable machine-readable explanation of the coverage verdict. + Reason string `json:"reason"` } // CacheBackendRemoteStorageStatus reports the optional shared L3 independently @@ -881,10 +968,11 @@ type CacheBackendStatus struct { // resolves each replica to its engine pod by (tenant, replica_id) and then // attributes it to the owning CacheBackend — either via the engine pod's // `inferencecache.io/injected-by` annotation (the authoritative wiring -// signal stamped by the pod webhook) or, for pods that bypassed the -// webhook, via a deterministic first-match on `spec.engineSelector. -// matchLabels`. The poller writes write-only-on-change and never clears -// it on a single failed scrape (soft state). +// signal stamped by the pod webhook) or, for manually attached subscriber Pods +// that bypassed the webhook, via a deterministic metadata.name-ordered selector +// fallback. Admission rejects ambiguous ownership; the poller +// writes write-only-on-change and never clears it on a single failed scrape +// (soft state). type CacheBackendIndexParticipation struct { // PrefixCount is the sum of distinct prefix entries currently attributed // to this backend's replicas. Zero is a valid observed value — it means diff --git a/api/v1alpha1/cachebackend_types_test.go b/api/v1alpha1/cachebackend_types_test.go index b8641fd1..70606466 100644 --- a/api/v1alpha1/cachebackend_types_test.go +++ b/api/v1alpha1/cachebackend_types_test.go @@ -86,9 +86,37 @@ func TestCacheBackendCRDSchemaFieldsAndEnums(t *testing.T) { requireMinimum(t, mustProperty(t, podLocalServerSchema, "maxWorkers"), 1) nodeLocalSchema := mustProperty(t, lmCacheSchema, "nodeLocal") requireRequired(t, nodeLocalSchema, "server") + requireRequired(t, nodeLocalSchema, "idleRetentionSeconds") + idleRetentionSchema := mustProperty(t, nodeLocalSchema, "idleRetentionSeconds") + requireMinimum(t, idleRetentionSchema, 0) + requireMaximum(t, idleRetentionSchema, 86400) + if got := idleRetentionSchema["default"]; got != float64(300) { + t.Fatalf("nodeLocal.idleRetentionSeconds default = %v, want 300", got) + } nodeLocalServerSchema := mustProperty(t, nodeLocalSchema, "server") + for _, field := range []string{"image", "port", "httpPort", "l1Capacity", "maxGPUWorkers", "maxCPUWorkers", "resources"} { + requireRequired(t, nodeLocalServerSchema, field) + } + for _, field := range []string{"port", "httpPort"} { + requireMinimum(t, mustProperty(t, nodeLocalServerSchema, field), 1) + requireMaximum(t, mustProperty(t, nodeLocalServerSchema, field), 65535) + } requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxGPUWorkers"), 1) requireMinimum(t, mustProperty(t, nodeLocalServerSchema, "maxCPUWorkers"), 1) + nodeLocalSchedulingSchema := mustProperty(t, nodeLocalSchema, "scheduling") + for _, field := range []string{ + "tolerations", "imagePullSecrets", "serviceAccountName", + "securityContext", "priorityClassName", "schedulerName", "runtimeClassName", + "terminationGracePeriodSeconds", + } { + if !hasProperty(nodeLocalSchedulingSchema, field) { + t.Fatalf("spec.lmCache.nodeLocal.scheduling.%s is missing from CRD schema", field) + } + } + for _, field := range []string{"nodeSelector", "affinity"} { + requireNoProperty(t, nodeLocalSchedulingSchema, field) + } + requireMinimum(t, mustProperty(t, nodeLocalSchedulingSchema, "terminationGracePeriodSeconds"), 0) requireMinimum(t, mustProperty(t, lmCacheSchema, "chunkSizeTokens"), 1) remoteStorageSchema := mustProperty(t, specSchema, "remoteStorage") requireRequired(t, remoteStorageSchema, "provider") @@ -236,8 +264,7 @@ func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { }, }}, NodeLocal: &LMCacheNodeLocalSpec{Scheduling: &LMCacheNodeLocalSchedulingSpec{ - NodeSelector: map[string]string{"pool": "cache"}, - Tolerations: []corev1.Toleration{{Key: "cache"}}, + Tolerations: []corev1.Toleration{{Key: "cache"}}, }}, }, RemoteStorage: &CacheBackendRemoteStorageSpec{ @@ -263,6 +290,7 @@ func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { ReadyEnginePods: 1, DesiredServers: 2, ReadyServers: 1, + EnginePodCoverage: []CacheBackendEnginePodCoverageStatus{{Name: "engine-0", NodeName: "node-a", Ready: true, Covered: true, Reason: "ConnectorReady"}}, }, RemoteStorage: &CacheBackendRemoteStorageStatus{ Provider: CacheBackendRemoteStorageProviderRedis, @@ -286,7 +314,6 @@ func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { copied := backend.DeepCopy() backend.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("64Gi") - backend.Spec.LMCache.NodeLocal.Scheduling.NodeSelector["pool"] = "general" backend.Spec.LMCache.NodeLocal.Scheduling.Tolerations[0].Key = "general" *backend.Spec.RemoteStorage.Redis.Database = 9 backend.Spec.RemoteStorage.Redis.Authentication.Password.Name = "changed" @@ -296,7 +323,7 @@ func TestCacheBackendMPRoundTripAndDeepCopy(t *testing.T) { if got := copied.Spec.LMCache.PodLocal.Server.Resources.Requests[corev1.ResourceMemory]; got.Cmp(resource.MustParse("33Gi")) != 0 { t.Fatalf("podLocal server resources alias original: %s", got.String()) } - if copied.Spec.LMCache.NodeLocal.Scheduling.NodeSelector["pool"] != "cache" || copied.Spec.LMCache.NodeLocal.Scheduling.Tolerations[0].Key != "cache" { + if copied.Spec.LMCache.NodeLocal.Scheduling.Tolerations[0].Key != "cache" { t.Fatalf("nodeLocal scheduling was not deep-copied") } if *copied.Spec.RemoteStorage.Redis.Database != 2 || copied.Spec.RemoteStorage.Redis.Authentication.Password.Name != "redis-auth" { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5de0b0b8..75660aca 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CacheBackend) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendConnectorStatus) DeepCopyInto(out *CacheBackendConnectorStatus) { *out = *in + if in.EnginePodCoverage != nil { + in, out := &in.EnginePodCoverage, &out.EnginePodCoverage + *out = make([]CacheBackendEnginePodCoverageStatus, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendConnectorStatus. @@ -58,6 +63,21 @@ func (in *CacheBackendConnectorStatus) DeepCopy() *CacheBackendConnectorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CacheBackendEnginePodCoverageStatus) DeepCopyInto(out *CacheBackendEnginePodCoverageStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CacheBackendEnginePodCoverageStatus. +func (in *CacheBackendEnginePodCoverageStatus) DeepCopy() *CacheBackendEnginePodCoverageStatus { + if in == nil { + return nil + } + out := new(CacheBackendEnginePodCoverageStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CacheBackendEngineSelector) DeepCopyInto(out *CacheBackendEngineSelector) { *out = *in @@ -338,7 +358,7 @@ func (in *CacheBackendStatus) DeepCopyInto(out *CacheBackendStatus) { if in.Connector != nil { in, out := &in.Connector, &out.Connector *out = new(CacheBackendConnectorStatus) - **out = **in + (*in).DeepCopyInto(*out) } if in.RemoteStorage != nil { in, out := &in.RemoteStorage, &out.RemoteStorage @@ -869,13 +889,6 @@ func (in *LMCacheEngineSpec) DeepCopy() *LMCacheEngineSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LMCacheNodeLocalSchedulingSpec) DeepCopyInto(out *LMCacheNodeLocalSchedulingSpec) { *out = *in - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in)) @@ -883,11 +896,26 @@ func (in *LMCacheNodeLocalSchedulingSpec) DeepCopyInto(out *LMCacheNodeLocalSche (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Affinity != nil { - in, out := &in.Affinity, &out.Affinity - *out = new(v1.Affinity) + if in.ImagePullSecrets != nil { + in, out := &in.ImagePullSecrets, &out.ImagePullSecrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } + if in.SecurityContext != nil { + in, out := &in.SecurityContext, &out.SecurityContext + *out = new(v1.PodSecurityContext) (*in).DeepCopyInto(*out) } + if in.RuntimeClassName != nil { + in, out := &in.RuntimeClassName, &out.RuntimeClassName + *out = new(string) + **out = **in + } + if in.TerminationGracePeriodSeconds != nil { + in, out := &in.TerminationGracePeriodSeconds, &out.TerminationGracePeriodSeconds + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LMCacheNodeLocalSchedulingSpec. diff --git a/config/crd/bases/inferencecache.io_cachebackends.yaml b/config/crd/bases/inferencecache.io_cachebackends.yaml index 2d99b86d..1585b340 100644 --- a/config/crd/bases/inferencecache.io_cachebackends.yaml +++ b/config/crd/bases/inferencecache.io_cachebackends.yaml @@ -74,11 +74,16 @@ spec: type: boolean engineSelector: description: |- - EngineSelector selects which engine pods this CacheBackend claims via - equality-based label matching over the pod's labels: every key/value - in MatchLabels must be present on the pod. The full - metav1.LabelSelector surface (matchExpressions, operator-based - selection) is NOT exposed today — only MatchLabels. + EngineSelector selects which engine pods this CacheBackend claims. A + non-empty selector must contain exactly one MatchLabels entry whose key is + inferencecache.io/cache-domain. The value is an operator-chosen, + namespace-scoped compatibility-domain ID; the selected Pods must carry the + same label. Pods may carry other labels, but they do not participate in + CacheBackend ownership. The full metav1.LabelSelector surface + (matchExpressions, operator-based selection) is NOT exposed today. + Admission requires the domain value to be unique among CacheBackends in + the same namespace. Every engine Pod must have exactly one CacheBackend + owner. Pods that match get runtime-adapter engine wiring injected by the mutating Pod admission webhook at pod CREATE time. LMCache MP adapters inject the local connector immediately; when an optional managed Redis @@ -468,948 +473,301 @@ spec: type: integer nodeLocal: description: |- - NodeLocal configures a future per-node MP server. Admission currently - rejects this block so it can never be accepted as inert configuration. + NodeLocal configures a controller-owned, engine-demand-driven per-node MP + server pool. properties: + idleRetentionSeconds: + default: 300 + description: |- + IdleRetentionSeconds keeps an otherwise healthy per-node server alive + after the final selected engine leaves that node. A new matching engine + scheduled there during the window reuses the same server Pod and shared + L1. Zero requests immediate deletion. + format: int32 + maximum: 86400 + minimum: 0 + type: integer scheduling: description: |- - LMCacheNodeLocalSchedulingSpec configures placement of the future per-node - MP server workload. + Scheduling optionally overrides server-Pod runtime and operational fields. + It cannot select nodes; inference-engine scheduling remains authoritative. properties: - affinity: - description: Affinity is a group of affinity scheduling - rules. + imagePullSecrets: + items: + description: |- + LocalObjectReference contains enough information to let you locate the + referenced object inside the same namespace. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + type: array + priorityClassName: + type: string + runtimeClassName: + description: |- + RuntimeClassName overrides the runtime inherited from the engine Pod used + to place this server. Clusters that do not make the NVIDIA runtime the + default can use this field to provide GPU visibility without reserving + allocatable GPUs. + type: string + schedulerName: + type: string + securityContext: + description: |- + PodSecurityContext holds pod-level security attributes and common container settings. + Some fields are also present in container.securityContext. Field values of + container.securityContext take precedence over field values of PodSecurityContext. properties: - nodeAffinity: - description: Describes node affinity scheduling rules - for the pod. + appArmorProfile: + description: |- + appArmorProfile is the AppArmor options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: - preferredDuringSchedulingIgnoredDuringExecution: + localhostProfile: description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node matches the corresponding matchExpressions; the - node(s) with the highest sum are the most preferred. - items: - description: |- - An empty preferred scheduling term matches all objects with implicit weight 0 - (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). - properties: - preference: - description: A node selector term, associated - with the corresponding weight. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - weight: - description: Weight associated with matching - the corresponding nodeSelectorTerm, in - the range 1-100. - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: + localhostProfile indicates a profile loaded on the node that should be used. + The profile must be preconfigured on the node to work. + Must match the loaded name of the profile. + Must be set if and only if type is "Localhost". + type: string + type: description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to an update), the system - may or may not try to eventually evict the pod from its node. - properties: - nodeSelectorTerms: - description: Required. A list of node selector - terms. The terms are ORed. - items: - description: |- - A null or empty node selector term matches no objects. The requirements of - them are ANDed. - The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. - properties: - matchExpressions: - description: A list of node selector - requirements by node's labels. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchFields: - description: A list of node selector - requirements by node's fields. - items: - description: |- - A node selector requirement is a selector that contains values, a key, and an operator - that relates the key and values. - properties: - key: - description: The label key that - the selector applies to. - type: string - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - type: object - x-kubernetes-map-type: atomic - type: array - x-kubernetes-list-type: atomic - required: - - nodeSelectorTerms - type: object - x-kubernetes-map-type: atomic + type indicates which kind of AppArmor profile will be applied. + Valid options are: + Localhost - a profile pre-loaded on the node. + RuntimeDefault - the container runtime's default profile. + Unconfined - no AppArmor enforcement. + type: string + required: + - type type: object - podAffinity: - description: Describes pod affinity scheduling rules - (e.g. co-locate this pod in the same node, zone, - etc. as some other pod(s)). - properties: - preferredDuringSchedulingIgnoredDuringExecution: - description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling affinity expressions, etc.), - compute a sum by iterating through the elements of this field and adding - "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added per-node - to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: - description: |- - If the affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic + fsGroup: + description: |- + A special supplemental group that applies to all containers in a pod. + Some volume types allow the Kubelet to change the ownership of that volume + to be owned by the pod: + + 1. The owning GID will be the FSGroup + 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) + 3. The permission bits are OR'd with rw-rw---- + + If unset, the Kubelet will not modify the ownership and permissions of any volume. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + fsGroupChangePolicy: + description: |- + fsGroupChangePolicy defines behavior of changing ownership and permission of the volume + before being exposed inside Pod. This field will only apply to + volume types which support fsGroup based ownership(and permissions). + It will have no effect on ephemeral volume types such as: secret, configmaps + and emptydir. + Valid values are "OnRootMismatch" and "Always". If not specified, "Always" is used. + Note that this field cannot be set when spec.os.name is windows. + type: string + runAsGroup: + description: |- + The GID to run the entrypoint of the container process. + Uses runtime default if unset. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + runAsNonRoot: + description: |- + Indicates that the container must run as a non-root user. + If true, the Kubelet will validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start the container if it does. + If unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: |- + The UID to run the entrypoint of the container process. + Defaults to user specified in image metadata if unspecified. + May also be set in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence + for that container. + Note that this field cannot be set when spec.os.name is windows. + format: int64 + type: integer + seLinuxChangePolicy: + description: |- + seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. + It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. + Valid values are "MountOption" and "Recursive". + + "Recursive" means relabeling of all files on all Pod volumes by the container runtime. + This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node. + + "MountOption" mounts all eligible Pod volumes with `-o context` mount option. + This requires all Pods that share the same volume to use the same SELinux label. + It is not possible to share the same volume among privileged and unprivileged Pods. + Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes + whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their + CSIDriver instance. Other volumes are always re-labelled recursively. + "MountOption" value is allowed only when SELinuxMount feature gate is enabled. + + If not specified and SELinuxMount feature gate is enabled, "MountOption" is used. + If not specified and SELinuxMount feature gate is disabled, "MountOption" is used for ReadWriteOncePod volumes + and "Recursive" for all other volumes. + + This field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers. + + All Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. + Note that this field cannot be set when spec.os.name is windows. + type: string + seLinuxOptions: + description: |- + The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a random SELinux context for each + container. May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + Note that this field cannot be set when spec.os.name is windows. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string type: object - podAntiAffinity: - description: Describes pod anti-affinity scheduling - rules (e.g. avoid putting this pod in the same node, - zone, etc. as some other pod(s)). + seccompProfile: + description: |- + The seccomp options to use by the containers in this pod. + Note that this field cannot be set when spec.os.name is windows. properties: - preferredDuringSchedulingIgnoredDuringExecution: + localhostProfile: description: |- - The scheduler will prefer to schedule pods to nodes that satisfy - the anti-affinity expressions specified by this field, but it may choose - a node that violates one or more of the expressions. The node that is - most preferred is the one with the greatest sum of weights, i.e. - for each node that meets all of the scheduling requirements (resource - request, requiredDuringScheduling anti-affinity expressions, etc.), - compute a sum by iterating through the elements of this field and subtracting - "weight" from the sum if the node has pods which matches the corresponding podAffinityTerm; the - node(s) with the highest sum are the most preferred. - items: - description: The weights of all of the matched - WeightedPodAffinityTerm fields are added per-node - to find the most preferred node(s) - properties: - podAffinityTerm: - description: Required. A pod affinity term, - associated with the corresponding weight. - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is - a list of label selector requirements. - The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - weight: - description: |- - weight associated with matching the corresponding podAffinityTerm, - in the range 1-100. - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - x-kubernetes-list-type: atomic - requiredDuringSchedulingIgnoredDuringExecution: + localhostProfile indicates a profile defined in a file on the node should be used. + The profile must be preconfigured on the node to work. + Must be a descending path, relative to the kubelet's configured seccomp profile location. + Must be set if type is "Localhost". Must NOT be set for any other type. + type: string + type: description: |- - If the anti-affinity requirements specified by this field are not met at - scheduling time, the pod will not be scheduled onto the node. - If the anti-affinity requirements specified by this field cease to be met - at some point during pod execution (e.g. due to a pod label update), the - system may or may not try to eventually evict the pod from its node. - When there are multiple elements, the lists of nodes corresponding to each - podAffinityTerm are intersected, i.e. all terms must be satisfied. - items: - description: |- - Defines a set of pods (namely those matching the labelSelector - relative to the given namespace(s)) that this pod should be - co-located (affinity) or not co-located (anti-affinity) with, - where co-located is defined as running on a node whose value of - the label with key matches that of any node on which - a pod of the set of pods is running - properties: - labelSelector: - description: |- - A label query over a set of resources, in this case pods. - If it's null, this PodAffinityTerm matches with no Pods. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - matchLabelKeys: - description: |- - MatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both matchLabelKeys and labelSelector. - Also, matchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - mismatchLabelKeys: - description: |- - MismatchLabelKeys is a set of pod label keys to select which pods will - be taken into consideration. The keys are used to lookup values from the - incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` - to select the group of existing pods which pods will be taken into consideration - for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming - pod labels will be ignored. The default value is empty. - The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. - Also, mismatchLabelKeys cannot be set when labelSelector isn't set. - items: - type: string - type: array - x-kubernetes-list-type: atomic - namespaceSelector: - description: |- - A label query over the set of namespaces that the term applies to. - The term is applied to the union of the namespaces selected by this field - and the ones listed in the namespaces field. - null selector and null or empty namespaces list means "this pod's namespace". - An empty selector ({}) matches all namespaces. - properties: - matchExpressions: - description: matchExpressions is a list - of label selector requirements. The - requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label - key that the selector applies - to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - namespaces: - description: |- - namespaces specifies a static list of namespace names that the term applies to. - The term is applied to the union of the namespaces listed in this field - and the ones selected by namespaceSelector. - null or empty namespaces list and null namespaceSelector means "this pod's namespace". - items: - type: string - type: array - x-kubernetes-list-type: atomic - topologyKey: - description: |- - This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching - the labelSelector in the specified namespaces, where co-located is defined as running on a node - whose value of the label with key topologyKey matches that of any node on which any of the - selected pods is running. - Empty topologyKey is not allowed. - type: string - required: - - topologyKey - type: object - type: array - x-kubernetes-list-type: atomic + type indicates which kind of seccomp profile will be applied. + Valid options are: + + Localhost - a profile defined in a file on the node should be used. + RuntimeDefault - the container runtime default profile should be used. + Unconfined - no profile should be applied. + type: string + required: + - type + type: object + supplementalGroups: + description: |- + A list of groups applied to the first process run in each container, in + addition to the container's primary GID and fsGroup (if specified). If + the SupplementalGroupsPolicy feature is enabled, the + supplementalGroupsPolicy field determines whether these are in addition + to or instead of any group memberships defined in the container image. + If unspecified, no additional groups are added, though group memberships + defined in the container image may still be used, depending on the + supplementalGroupsPolicy field. + Note that this field cannot be set when spec.os.name is windows. + items: + format: int64 + type: integer + type: array + x-kubernetes-list-type: atomic + supplementalGroupsPolicy: + description: |- + Defines how supplemental groups of the first container processes are calculated. + Valid values are "Merge" and "Strict". If not specified, "Merge" is used. + (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled + and the container runtime must implement support for this feature. + Note that this field cannot be set when spec.os.name is windows. + type: string + sysctls: + description: |- + Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported + sysctls (by the container runtime) might fail to launch. + Note that this field cannot be set when spec.os.name is windows. + items: + description: Sysctl defines a kernel parameter to + be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + windowsOptions: + description: |- + The Windows specific settings applied to all containers. + If unspecified, the options within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + Note that this field cannot be set when spec.os.name is linux. + properties: + gmsaCredentialSpec: + description: |- + GMSACredentialSpec is where the GMSA admission webhook + (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the + GMSA credential spec named by the GMSACredentialSpecName field. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. + type: string + hostProcess: + description: |- + HostProcess determines if a container should be run as a 'Host Process' container. + All of a Pod's containers must have the same effective HostProcess value + (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). + In addition, if HostProcess is true then HostNetwork must also be set to true. + type: boolean + runAsUserName: + description: |- + The UserName in Windows to run the entrypoint of the container process. + Defaults to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext takes precedence. + type: string type: object type: object - nodeSelector: - additionalProperties: - type: string - type: object + serviceAccountName: + type: string + terminationGracePeriodSeconds: + format: int64 + minimum: 0 + type: integer tolerations: + description: |- + Tolerations are merged with the tolerations of an engine already running + on the target node. They allow the server Pod to pass that node's taints + without selecting a different node. items: description: |- The pod this Toleration is attached to tolerates any taint that matches @@ -1451,16 +809,30 @@ spec: type: object server: description: |- - LMCacheNodeLocalServerSpec describes the future one-server-per-node MP - topology. The shape is published now so the API does not need another - topology redesign, but admission rejects NodeLocal until Phase 8. + LMCacheNodeLocalServerSpec configures the controller-owned LMCache MP server + shared by selected engine Pods on one node chosen by the inference system. properties: + httpPort: + description: |- + HTTPPort is the node-bound FastAPI health/control port used by probes and + by the engine startup gate to verify the same-node server identity. + format: int32 + maximum: 65535 + minimum: 1 + type: integer image: + description: |- + Image is the digest-pinned LMCache server image. The same image is used + by the lightweight engine startup gate; CacheBackend never changes the + inference-engine image. type: string l1Capacity: anyOf: - type: integer - type: string + description: |- + L1Capacity is one shared host-memory budget per active engine node, not per + selected engine Pod. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true x-kubernetes-validations: @@ -1472,19 +844,22 @@ spec: minimum: 1 type: integer maxGPUWorkers: - description: MaxGPUWorkers bounds workers serving GPU-backed - engine clients. + description: |- + MaxGPUWorkers bounds workers serving GPU-backed engine clients and must + cover the maximum number of engine instances expected on one node. format: int32 minimum: 1 type: integer port: + description: Port is the node-bound LMCache MP data port. format: int32 maximum: 65535 minimum: 1 type: integer resources: - description: ResourceRequirements describes the compute - resource requirements. + description: |- + Resources are applied to every per-node server Pod. Admission requires + memory request and limit headroom above the shared L1 budget. properties: claims: description: |- @@ -1544,6 +919,7 @@ spec: type: object type: object required: + - httpPort - image - l1Capacity - maxCPUWorkers @@ -1552,6 +928,7 @@ spec: - resources type: object required: + - idleRetentionSeconds - server type: object podLocal: @@ -1666,9 +1043,8 @@ spec: - server type: object topology: - description: |- - Topology selects the canonical LMCache MP server placement. PodLocal is - implemented first; NodeLocal is reserved and rejected until Phase 8. + description: Topology selects the canonical LMCache MP server + placement. enum: - PodLocal - NodeLocal @@ -3410,10 +2786,47 @@ spec: desiredServers: description: |- DesiredServers is one per selected engine Pod for PodLocal and one per - eligible node for NodeLocal. + distinct active scheduled engine node for NodeLocal. format: int32 minimum: 0 type: integer + enginePodCoverage: + description: |- + EnginePodCoverage reports the connector verdict for every active selected + engine Pod. The list is keyed by Pod name and sorted by name by the + controller so operators can identify the exact uncovered instance. + items: + description: |- + CacheBackendEnginePodCoverageStatus is the per-engine connector verdict. + Ready means the engine Pod itself is Ready; Covered means the current + CacheBackend generation has exactly one healthy reachable MP server for it. + properties: + covered: + type: boolean + name: + description: Name is the selected engine Pod name in the + CacheBackend namespace. + type: string + nodeName: + description: NodeName is the scheduled node. It is empty + while the Pod is pending. + type: string + ready: + type: boolean + reason: + description: Reason is a stable machine-readable explanation + of the coverage verdict. + type: string + required: + - covered + - name + - ready + - reason + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map matchedEnginePods: description: MatchedEnginePods is the number of selected engine Pods observed. diff --git a/config/observability/lmcache-podmonitor.yaml b/config/observability/lmcache-podmonitor.yaml index e6afe524..3658c573 100644 --- a/config/observability/lmcache-podmonitor.yaml +++ b/config/observability/lmcache-podmonitor.yaml @@ -2,10 +2,11 @@ # # SPDX-License-Identifier: Apache-2.0 -# PodMonitor for successfully injected PodLocal LMCache multiprocess native -# sidecars. The mutating webhook stamps the selector label only after the typed -# MP renderer has completed atomically, so matching Pods have the named -# `lmcache-http` port and the real FastAPI `/metrics` route. +# PodMonitor for PodLocal native sidecars and controller-owned NodeLocal server +# Pods. Both expose the real FastAPI `/metrics` route on the named +# `lmcache-http` port. NodeLocal targets use hostNetwork, so NetworkPolicy does +# not isolate this unauthenticated listener; restrict its host port with node +# firewall controls inside the backend's trust domain. # # Engine Pods can live in any workload namespace while this PodMonitor remains # in the inference-cache system/monitoring namespace. `namespaceSelector.any` diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 88b51572..940b7ee9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -9,8 +9,11 @@ rules: resources: - pods verbs: + - create + - delete - get - list + - patch - watch - apiGroups: - "" diff --git a/config/samples/README.md b/config/samples/README.md index 61aeff2b..7bf107d5 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -23,8 +23,12 @@ multi-tenant, Namespaces): [host-only](cachebackend-vllm-podlocal-host-only.yaml), [managed Redis](cachebackend-vllm-podlocal-managed-redis.yaml), and [external Redis](cachebackend-vllm-podlocal-external-redis.yaml). All LMCache - offload samples use the typed PodLocal MP API; `EventsOnly` intentionally - carries no LMCache data plane. + offload samples use the typed MP API. NodeLocal host-only profiles are + available for [vLLM](cachebackend-vllm-nodelocal-host-only.yaml) and + [SGLang](cachebackend-sglang-nodelocal-host-only.yaml); the inference system + owns their placement and they opt into server host networking plus shared + host `/dev/shm`. + `EventsOnly` intentionally carries no LMCache data plane. ## Recipe catalog @@ -63,6 +67,20 @@ degrades to `NoKVEventsObserved`. Externally owned backends are exempt from that they go `Ready` as soon as admission accepts the endpoint. See the [quickstart](../../docs/quickstart.md). +NodeLocal focused samples are not five-minute recipes. Before applying one, +reserve its MP and HTTP host ports on every node where the inference system may +place a selected engine, and ensure all selected engine Pods belong to one +mutually trusted tenant domain. CacheBackend does not select nodes or rewrite +engine placement. The declared `l1Capacity` is a shared budget on every active +engine node; `maxGPUWorkers` must cover the maximum selected engine instances on +one node. `idleRetentionSeconds` keeps the per-node server and its L1 warm after +the final engine leaves (300 seconds in the focused samples; zero deletes it +immediately). The server requests no allocatable GPU, so set the optional +`nodeLocal.scheduling.runtimeClassName` when the engine's inherited runtime does +not provide the required NVIDIA visibility. +Host-network listeners bypass Kubernetes NetworkPolicy, so restrict them with +node firewall controls. Do not add a load-balanced Service for the MP port. + `SGLangHiCache` is endpoint-free and has no endpoint publication race. Its first implementation intentionally publishes no `Ready` condition; the matching Pod's injection annotations are the available wiring signal until the diff --git a/config/samples/cache_v1alpha1_cachebackend.yaml b/config/samples/cache_v1alpha1_cachebackend.yaml index 5d5a1d52..9f8cef3f 100644 --- a/config/samples/cache_v1alpha1_cachebackend.yaml +++ b/config/samples/cache_v1alpha1_cachebackend.yaml @@ -23,7 +23,7 @@ spec: type: LMCache engineSelector: matchLabels: - inferencecache.io/cache-enabled: "true" + inferencecache.io/cache-domain: cachebackend-sample lmCache: topology: PodLocal podLocal: diff --git a/config/samples/cachebackend-cpu-override.yaml b/config/samples/cachebackend-cpu-override.yaml index 60923110..be4edc99 100644 --- a/config/samples/cachebackend-cpu-override.yaml +++ b/config/samples/cachebackend-cpu-override.yaml @@ -41,7 +41,7 @@ spec: value: bar engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-cpu-override lmCache: topology: PodLocal chunkSizeTokens: 512 diff --git a/config/samples/cachebackend-events-only.yaml b/config/samples/cachebackend-events-only.yaml index 64cbde63..39ad02eb 100644 --- a/config/samples/cachebackend-events-only.yaml +++ b/config/samples/cachebackend-events-only.yaml @@ -54,7 +54,7 @@ spec: mode: EventsOnly engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-events-only observation: # Served model identifier the matched engine pods are loaded with. Plumbed # to the auto-attached kvevent-subscriber sidecar's --model-id so the index diff --git a/config/samples/cachebackend-external.yaml b/config/samples/cachebackend-external.yaml index 03050ec3..e0d8adfc 100644 --- a/config/samples/cachebackend-external.yaml +++ b/config/samples/cachebackend-external.yaml @@ -24,7 +24,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-external-redis lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-lmcache.yaml b/config/samples/cachebackend-lmcache.yaml index 053041d4..3e4268f3 100644 --- a/config/samples/cachebackend-lmcache.yaml +++ b/config/samples/cachebackend-lmcache.yaml @@ -21,7 +21,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-lmcache lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-sglang-hicache.yaml b/config/samples/cachebackend-sglang-hicache.yaml index 113e57e5..760d687c 100644 --- a/config/samples/cachebackend-sglang-hicache.yaml +++ b/config/samples/cachebackend-sglang-hicache.yaml @@ -11,7 +11,7 @@ spec: type: SGLangHiCache engineSelector: matchLabels: - app: sglang + inferencecache.io/cache-domain: sglang-hicache hiCache: ratio: "2.0" writePolicy: write_through diff --git a/config/samples/cachebackend-sglang-host-only.yaml b/config/samples/cachebackend-sglang-host-only.yaml index 399c4620..ebc0d3d7 100644 --- a/config/samples/cachebackend-sglang-host-only.yaml +++ b/config/samples/cachebackend-sglang-host-only.yaml @@ -15,7 +15,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app.kubernetes.io/name: sglang + inferencecache.io/cache-domain: sglang-host-only lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-sglang-nodelocal-host-only.yaml b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml new file mode 100644 index 00000000..4299fd2b --- /dev/null +++ b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed SGLang NodeLocal LMCache MP. SGLang engine Pods must explicitly declare +# --page-size as a divisor of chunkSizeTokens and use TP=1 for the currently +# validated profile. The controller follows selected engines onto their scheduled +# nodes and never rewrites engine placement. The 5556/8081 host-port pair is +# intentionally disjoint from the sibling vLLM sample if both backends have +# engines on the same node. Set scheduling.runtimeClassName when the engine's +# runtime does not provide the server's required NVIDIA visibility. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: sglang-nodelocal-host-only +spec: + runtime: SGLang + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/cache-domain: sglang-node-mp + lmCache: + topology: NodeLocal + chunkSizeTokens: 256 + nodeLocal: + idleRetentionSeconds: 300 + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5556 + httpPort: 8081 + l1Capacity: 32Gi + maxGPUWorkers: 4 + maxCPUWorkers: 4 + resources: + requests: + cpu: "2" + memory: 33Gi + limits: + cpu: "4" + memory: 34Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-sglang-podlocal-external-redis.yaml b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml index 1d935792..11f5bbed 100644 --- a/config/samples/cachebackend-sglang-podlocal-external-redis.yaml +++ b/config/samples/cachebackend-sglang-podlocal-external-redis.yaml @@ -18,7 +18,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: sglang-mp-external-redis + inferencecache.io/cache-domain: sglang-mp-external-redis lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-sglang-podlocal-host-only.yaml b/config/samples/cachebackend-sglang-podlocal-host-only.yaml index da72ca6c..ab17e6e4 100644 --- a/config/samples/cachebackend-sglang-podlocal-host-only.yaml +++ b/config/samples/cachebackend-sglang-podlocal-host-only.yaml @@ -17,7 +17,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: sglang-mp + inferencecache.io/cache-domain: sglang-mp lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml index 029d8660..246de161 100644 --- a/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml +++ b/config/samples/cachebackend-sglang-podlocal-managed-redis.yaml @@ -15,7 +15,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: sglang-mp-managed-redis + inferencecache.io/cache-domain: sglang-mp-managed-redis lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-sglang.yaml b/config/samples/cachebackend-sglang.yaml index 770a9874..bd86ed94 100644 --- a/config/samples/cachebackend-sglang.yaml +++ b/config/samples/cachebackend-sglang.yaml @@ -5,8 +5,8 @@ # Typed SGLang PodLocal LMCache MP with a controller-managed Redis L3. The # webhook injects the common lmcache-mp-server native sidecar plus SGLang's # --enable-lmcache/--lmcache-config-file launch surface. Kubernetes 1.29 or -# newer is required for native sidecars. SGLang TP>1 is outside the validated -# migration baseline. +# newer is required for native sidecars. SGLang TP>1 is outside the currently +# validated profile. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -20,7 +20,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app.kubernetes.io/name: sglang + inferencecache.io/cache-domain: sglang-lmcache lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-vllm-nodelocal-host-only.yaml b/config/samples/cachebackend-vllm-nodelocal-host-only.yaml new file mode 100644 index 00000000..f536a8a6 --- /dev/null +++ b/config/samples/cachebackend-vllm-nodelocal-host-only.yaml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2026 The inference-cache Authors +# +# SPDX-License-Identifier: Apache-2.0 + +# Typed vLLM NodeLocal LMCache MP with one on-demand shared server on every node +# that actually hosts a selected engine. The inference system remains the +# scheduling authority; CacheBackend does not select nodes or change engine +# placement. The server uses hostNetwork and host /dev/shm; node firewall policy +# must restrict the unauthenticated MP/HTTP host ports. CacheBackend does not +# install LMCache into or replace the engine image. If NVIDIA is not the +# engine/server runtime, set nodeLocal.scheduling.runtimeClassName. +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: vllm-nodelocal-host-only +spec: + runtime: VLLM + type: LMCache + integration: + role: ReadWrite + engineSelector: + matchLabels: + inferencecache.io/cache-domain: vllm-node-mp + lmCache: + topology: NodeLocal + chunkSizeTokens: 256 + nodeLocal: + idleRetentionSeconds: 300 + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + httpPort: 8080 + l1Capacity: 32Gi + maxGPUWorkers: 4 + maxCPUWorkers: 4 + resources: + requests: + cpu: "2" + memory: 33Gi + limits: + cpu: "4" + memory: 34Gi + observation: + modelID: meta-llama/Meta-Llama-3-8B-Instruct diff --git a/config/samples/cachebackend-vllm-podlocal-external-redis.yaml b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml index 00fe5808..f89d2ce3 100644 --- a/config/samples/cachebackend-vllm-podlocal-external-redis.yaml +++ b/config/samples/cachebackend-vllm-podlocal-external-redis.yaml @@ -18,7 +18,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: vllm-mp-external-redis + inferencecache.io/cache-domain: vllm-mp-external-redis lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-vllm-podlocal-host-only.yaml b/config/samples/cachebackend-vllm-podlocal-host-only.yaml index f831119c..d0e77443 100644 --- a/config/samples/cachebackend-vllm-podlocal-host-only.yaml +++ b/config/samples/cachebackend-vllm-podlocal-host-only.yaml @@ -17,7 +17,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: vllm-mp + inferencecache.io/cache-domain: vllm-mp lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml index e42f1e64..c7a14eac 100644 --- a/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml +++ b/config/samples/cachebackend-vllm-podlocal-managed-redis.yaml @@ -15,7 +15,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - inferencecache.io/runtime: vllm-mp-managed-redis + inferencecache.io/cache-domain: vllm-mp-managed-redis lmCache: topology: PodLocal chunkSizeTokens: 256 diff --git a/config/samples/cachebackend-with-engine.yaml b/config/samples/cachebackend-with-engine.yaml index 794ae24a..fbd52eaf 100644 --- a/config/samples/cachebackend-with-engine.yaml +++ b/config/samples/cachebackend-with-engine.yaml @@ -3,7 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # Paired typed-MP sample: one CacheBackend and one matching vLLM Deployment. -# The label `app: qwen-demo` is the binding; at Pod CREATE the webhook injects +# The `inferencecache.io/cache-domain: qwen-demo` label is the binding; at Pod +# CREATE the webhook injects # an LMCache MP native sidecar and the vLLM LMCacheMPConnector JSON. This example # intentionally omits remoteStorage, so L1 is per Pod and no cross-Pod sharing # is claimed. The engine image remains inference-owner supplied: normal engine @@ -20,7 +21,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: qwen-demo # <-- binding key/value (1 of 2) + inferencecache.io/cache-domain: qwen-demo # <-- ownership domain (1 of 2) observation: # Served model the engine pods are loaded with. Plumbed to the # auto-attached kvevent-subscriber sidecar so the index keys @@ -57,7 +58,8 @@ spec: template: metadata: labels: - app: qwen-demo # <-- binding key/value (2 of 2) + app: qwen-demo + inferencecache.io/cache-domain: qwen-demo # <-- ownership domain (2 of 2) spec: containers: - name: vllm diff --git a/config/samples/cachebackend-with-override.yaml b/config/samples/cachebackend-with-override.yaml index 2b2b35b7..89224fe8 100644 --- a/config/samples/cachebackend-with-override.yaml +++ b/config/samples/cachebackend-with-override.yaml @@ -48,7 +48,7 @@ spec: value: DEBUG engineSelector: matchLabels: - app: qwen-demo + inferencecache.io/cache-domain: qwen-demo observation: # Served model identifier the matched engine pods are loaded with. # Plumbed to the auto-attached kvevent-subscriber sidecar's --model-id @@ -81,7 +81,8 @@ spec: # # The Deployment name is `qwen-demo-engine`, distinct from the CacheBackend # name (`qwen-demo`), because the CacheBackend reconciler stands up its own -# Binding to the CacheBackend goes through the `app: qwen-demo` label match in +# Binding to the CacheBackend goes through the +# `inferencecache.io/cache-domain: qwen-demo` label match in # `spec.engineSelector`, not through the Deployment name. # # CPU vLLM image and flags mirror docs/reference-stack/manifests/cpu-local/ @@ -104,6 +105,7 @@ spec: metadata: labels: app: qwen-demo + inferencecache.io/cache-domain: qwen-demo spec: # The image below is arm64-only; pin scheduling to an arm64 node so # an unmodified apply on a mixed-arch cluster does not schedule the diff --git a/config/samples/recipe-cpu-dev.yaml b/config/samples/recipe-cpu-dev.yaml index dd38aaab..58795046 100644 --- a/config/samples/recipe-cpu-dev.yaml +++ b/config/samples/recipe-cpu-dev.yaml @@ -37,7 +37,8 @@ # manifest and webhook mutation without a GPU; it does not claim that every # vLLM CPU image bundles the connector. Normal engine startup is authoritative. # -# What binds the two objects: the label `app: cpu-dev` appears in BOTH +# What binds the two objects: the `inferencecache.io/cache-domain: cpu-dev` +# label appears in BOTH # `CacheBackend.spec.engineSelector.matchLabels` AND the engine Deployment's # pod-template labels. The mutating Pod webhook intercepts each engine pod at # CREATE, finds the matching CacheBackend, and injects the LMCache engine @@ -60,7 +61,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: cpu-dev # <-- binding label (1 of 2) + inferencecache.io/cache-domain: cpu-dev # <-- ownership domain (1 of 2) lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -97,7 +98,8 @@ spec: template: metadata: labels: - app: cpu-dev # <-- binding label (2 of 2) + app: cpu-dev + inferencecache.io/cache-domain: cpu-dev # <-- ownership domain (2 of 2) spec: nodeSelector: kubernetes.io/arch: amd64 # arm64 hosts: change to `arm64` diff --git a/config/samples/recipe-external-cache.yaml b/config/samples/recipe-external-cache.yaml index 1b43daf9..aa2edcaf 100644 --- a/config/samples/recipe-external-cache.yaml +++ b/config/samples/recipe-external-cache.yaml @@ -44,7 +44,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: external-demo + inferencecache.io/cache-domain: external-demo lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -86,6 +86,7 @@ spec: metadata: labels: app: external-demo + inferencecache.io/cache-domain: external-demo spec: nodeSelector: kubernetes.io/arch: amd64 # arm64 hosts: change to `arm64` diff --git a/config/samples/recipe-gpu-production.yaml b/config/samples/recipe-gpu-production.yaml index 9bbcfbaf..a1b67a20 100644 --- a/config/samples/recipe-gpu-production.yaml +++ b/config/samples/recipe-gpu-production.yaml @@ -20,7 +20,8 @@ # * A CachePolicy tunes eviction for the namespace (see the CachePolicy doc # pointer in config/samples/README.md). # -# Binding works exactly as in recipe-cpu-dev.yaml: the `app: prod-llm` label on +# Binding works exactly as in recipe-cpu-dev.yaml: the +# `inferencecache.io/cache-domain: prod-llm` label on # the engine pod template matches `spec.engineSelector.matchLabels`. Same # binding remains CREATE-time; apply the CacheBackend before the engine # Deployment. See docs/concepts/cachebackend-engine-binding.md. @@ -60,7 +61,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: prod-llm + inferencecache.io/cache-domain: prod-llm lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -144,6 +145,7 @@ spec: metadata: labels: app: prod-llm + inferencecache.io/cache-domain: prod-llm spec: containers: - name: vllm diff --git a/config/samples/recipe-multi-tenant.yaml b/config/samples/recipe-multi-tenant.yaml index 9f345537..bc4d5f69 100644 --- a/config/samples/recipe-multi-tenant.yaml +++ b/config/samples/recipe-multi-tenant.yaml @@ -93,7 +93,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: tenant-engine + inferencecache.io/cache-domain: tenant-a-engine lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -125,7 +125,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: tenant-engine + inferencecache.io/cache-domain: tenant-b-engine lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -161,6 +161,7 @@ spec: metadata: labels: app: tenant-engine + inferencecache.io/cache-domain: tenant-a-engine spec: nodeSelector: kubernetes.io/arch: amd64 # arm64 hosts: change to `arm64` @@ -224,6 +225,7 @@ spec: metadata: labels: app: tenant-engine + inferencecache.io/cache-domain: tenant-b-engine spec: nodeSelector: kubernetes.io/arch: amd64 # arm64 hosts: change to `arm64` diff --git a/config/samples/recipe-tuning.yaml b/config/samples/recipe-tuning.yaml index 9322399f..6103c3a2 100644 --- a/config/samples/recipe-tuning.yaml +++ b/config/samples/recipe-tuning.yaml @@ -45,7 +45,7 @@ spec: value: DEBUG engineSelector: matchLabels: - app: tuning-demo + inferencecache.io/cache-domain: tuning-demo lmCache: topology: PodLocal chunkSizeTokens: 128 @@ -80,6 +80,7 @@ spec: metadata: labels: app: tuning-demo + inferencecache.io/cache-domain: tuning-demo spec: nodeSelector: kubernetes.io/arch: amd64 # arm64 hosts: change to `arm64` diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 7a6abbfd..717df19c 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -70,6 +70,7 @@ Every finding carries a stable, greppable code. Codes are permanent identifiers | `CB007` | WARN | `FunctionalProbeOK` condition present but not `True` — the controller's functional self-test is failing for this backend (explains a Ready downgrade) | | `EP001` | WARN | matched engine pod missing an injection marker (no `inferencecache.io/injected-by` annotation and no Event) | | `EP002` | OK | matched engine pod is injected (annotation or Event) | +| `EP003` | WARN | engine pod matches multiple same-namespace CacheBackends; no implicit owner is selected | | `OP001` | WARN | orphaned engine pod (NoMatchingCacheBackend; forward-looking — see note below) | | `CT001` | WARN | CacheTenant over quota (`QuotaExceeded=True`) | | `CT002` | OK | CacheTenant within quota | diff --git a/docs/concepts/cachebackend-engine-binding.md b/docs/concepts/cachebackend-engine-binding.md index ba2b49d5..87237cb1 100644 --- a/docs/concepts/cachebackend-engine-binding.md +++ b/docs/concepts/cachebackend-engine-binding.md @@ -7,13 +7,13 @@ engine-specific connector wire it adds. ## Current LMCache flow -For `spec.type: LMCache`, current manifests declare -`spec.lmCache.topology: PodLocal`. At Pod CREATE, the webhook: +For `spec.type: LMCache`, current manifests declare a typed PodLocal or +NodeLocal topology. At Pod CREATE, the webhook: -1. finds matching CacheBackends in the Pod's namespace; +1. finds the one matching CacheBackend in the Pod's namespace; 2. selects the runtime-specific MP adapter; -3. injects one `lmcache-mp-server` native sidecar, shared `/dev/shm`, and the - vLLM or SGLang connector launch surface; +3. injects the vLLM or SGLang connector launch surface plus either a PodLocal + native sidecar or a NodeLocal same-node startup gate and host `/dev/shm`; 4. optionally binds the MP server to a Redis L3; and 5. stamps `inferencecache.io/injected-by` and `inferencecache.io/injected-by-uid`. @@ -22,16 +22,48 @@ The engine image is never replaced or inspected. Normal engine initialization is the authoritative compatibility check for the required connector/package. PodLocal native sidecars require Kubernetes 1.29 or newer. +NodeLocal is engine-first. CacheBackend creation alone creates no server. After +the inference system schedules an injected engine Pod, the controller creates +one CacheBackend-owned server Pod for each distinct active engine node. The +server uses exact node-name affinity, so Kubernetes still evaluates taints, +resources, and declared host-port conflicts. The Downward API supplies the +engine's `status.hostIP`; no ClusterIP participates. The init gate blocks normal +engine startup until `/config` and `/healthcheck` verify the same +name/UID/generation and live server configuration. Each CacheBackend UID also +derives an explicit `lmcache_l1_pool_inferencecache_` POSIX SHM name; the +gate verifies both the declared and effective live name before starting the +engine. This prevents accidental unlink/rebind between co-located pools but is +ownership verification, not cryptographic authentication, so the +host-network/server pool and node-wide `/dev/shm` still require one trusted +tenant domain. +When the final selected engine leaves a node, the server enters the configured +`idleRetentionSeconds` window instead of being coupled to that engine Pod's +restart. Demand returning during the window clears the idle marker and reuses +the same server and L1; expiry removes the server. The default is 300 seconds, +and zero requests immediate deletion. + +Every new non-empty `engineSelector` contains exactly one label: +`inferencecache.io/cache-domain`. Its value is unique within the namespace and +identifies one runtime/model/KV-layout/trust compatibility domain. Engine Pods +may carry any other application, scheduling, and observability labels, but +those labels do not participate in CacheBackend ownership. CREATE and UPDATE +both enforce this shape. As a runtime backstop for concurrent CREATE admission, +the Pod webhook denies a Pod matching more than one CacheBackend; it never +chooses one by name. + ```text CacheBackend selector ──matches at Pod CREATE──▶ mutating webhook │ + PodLocal: native sidecar │ NodeLocal: startup gate ▼ -engine Pod: engine + LMCache MP server sidecar + optional subscriber - │ - └── optional RESP ──▶ Redis L3 + inference system schedules engine + │ + NodeLocal controller observes spec.nodeName + ▼ + one same-node server Pod per active node ``` -Host-only PodLocal objects publish no endpoint. With external Redis, the +Host-only PodLocal and NodeLocal objects publish no endpoint. With external Redis, the webhook uses `spec.remoteStorage.endpoint`; with managed Redis, it uses the controller-resolved endpoint. Connector readiness and remote-storage readiness are reported independently. @@ -41,9 +73,10 @@ are reported independently. 1. Apply the CacheBackend before creating engine Pods. 2. Create an engine Deployment whose Pod-template labels include every `spec.engineSelector.matchLabels` entry. -3. Admission injects the complete MP wire atomically. A collision or invalid - Pod shape fails open without a partial mutation; inspect Pod annotations, - Events, and engine startup logs. +3. Admission injects the complete MP wire atomically. Ordinary lookup or + adapter failures fail open without a partial mutation; selector ambiguity + is denied because choosing a cache trust domain is unsafe. Inspect Pod + annotations, Events, and engine startup logs. 4. If `--kvevent-subscriber-image` is configured and `spec.observation.modelID` is set, the webhook also adds the observation sidecar. The subscriber reports metadata-only KV events to the policy index. @@ -62,7 +95,7 @@ spec: type: LMCache engineSelector: matchLabels: - app: qwen-demo + inferencecache.io/cache-domain: qwen-demo lmCache: topology: PodLocal podLocal: @@ -91,6 +124,7 @@ spec: metadata: labels: app: qwen-demo + inferencecache.io/cache-domain: qwen-demo spec: containers: - name: vllm @@ -107,9 +141,10 @@ A fuller paired sample is | `MATCHED: 0` and no injection annotation | Selector and Pod labels differ. | Align the labels and recreate the Pod. | | A matching Pod has no injection annotation | Admission failed open because of an invalid/colliding Pod shape or an unavailable managed Redis endpoint. | Read webhook logs and Pod Events, fix the reported shape, then recreate the Pod. | | Engine crashes after successful injection | The runtime-owned image lacks a compatible LMCache client/API, or another engine startup requirement failed. | Inspect engine logs and use a compatible pinned image; CacheBackend does not replace it. | -| Multiple CacheBackends match one Pod | Selectors overlap; the lexicographically first CacheBackend wins. | Narrow selectors so every engine Pod has one owner. | +| Multiple CacheBackends could match one Pod | A cache-domain value was reused by concurrent creates. CacheBackend admission normally rejects the duplicate; the Pod webhook also denies an ambiguous live match rather than choosing a backend. | Give every CacheBackend a unique namespace-scoped `inferencecache.io/cache-domain` value and put that value on only the intended engine Pod templates. | +| NodeLocal engine stays in `lmcache-node-local-gate` | Its on-demand same-node server is not healthy, the host ports conflict, the effective UID-scoped SHM pool is unavailable/mismatched, or live config belongs to another backend. | Inspect the server Pod args, `/config`, scheduler events, and `status.connector.enginePodCoverage`; fix ports, host `/dev/shm` capacity, resources, or runtime configuration and recreate or reschedule. | | Pod was relabeled after creation | Admission is CREATE-only. | Recreate the Pod. | | Pod intentionally needs no cache injection | No explicit opt-out was set. | Put `inferencecache.io/skip-inject: "true"` on the Pod template and recreate it. | -Legacy topology-less vLLM/IP binding remains implemented only for Phase 7 -compatibility tests. It is not a current sample or recommended production path. +Legacy topology-less vLLM/IP binding exists only as negative search/schema +assertions. No production adapter implements it. diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index e47f2aa6..88da9718 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -3,8 +3,9 @@ Status: implemented · Tracks: InferenceCache tech spec §4.1 · API group: `inferencecache.io/v1alpha1` > **Current production contract:** LMCache uses typed -> `spec.lmCache.topology: PodLocal` multiprocess wiring for both vLLM and -> SGLang, with optional Redis selected explicitly. References to topology-less +> `spec.lmCache.topology: PodLocal|NodeLocal` multiprocess wiring for both vLLM +> and SGLang, with optional Redis selected explicitly. NodeLocal control-plane +> support and its required SJC GPU matrix were completed in Phase 8. References to topology-less > LMCacheServer, the former IP-wired Mooncake provider, `lm://`, and the IP > connector are explicitly marked history for behavior physically removed in > Phase 7. Mooncake remains a planned typed MP L2 provider; it is not available @@ -133,7 +134,7 @@ and [`config/samples/cachebackend-external.yaml`](../../config/samples/cacheback |---|---|---| | `runtime` | enum | Required inference runtime: `VLLM` or `SGLang`. Values are case-sensitive. | | `type` | enum | Engine-side cache implementation: `LMCache` or `SGLangHiCache`. Defaults to `LMCache`. | -| `lmCache` | object | Typed LMCache MP configuration: topology, chunk size, and PodLocal server image/port/L1/resources. | +| `lmCache` | object | Typed LMCache MP configuration: topology, chunk size, and PodLocal or NodeLocal server contract. | | `remoteStorage` | object | Optional remote tier. Omitting it means host-only and provisions no provider workload. | | `remoteStorage.provider` | enum | Current MP provider: `Redis`. | | `remoteStorage.ownership` | enum | `Managed` or `External`. | @@ -143,9 +144,9 @@ and [`config/samples/cachebackend-external.yaml`](../../config/samples/cacheback | `observation` | object | Observation-owned `modelID` and `firstEventTimeout`. | | `integration.mode` | enum | Which cache tiers the engine is wired for: `Offload` (default) or `EventsOnly`. `Offload` is full participation — cache-aware routing (tier-1) plus the KV-offload connector (tier-2). It may remain host-only, connect to externally owned remote storage, or provision a provider workload when `remoteStorage.ownership` is `Managed`. `EventsOnly` wires routing only: the kvevent-subscriber sidecar is injected when the controller runs with `--kvevent-subscriber-image` set and `observation.modelID` is present; otherwise the append is skipped fail-open. No KV connector or backend server is created. See [Events-only mode](#events-only-mode-specintegrationmode--eventsonly). | | `integration.role` | enum | Engine participation mode: `ReadOnly`, `WriteOnly`, or `ReadWrite`. Defaults to `ReadWrite`. LMCache currently admits only `ReadWrite`; directional roles remain reserved for a connector that demonstrably enforces them. | -| `integration.failOpen` | boolean | Default `true`. Remote L3 failure is soft by default and the PodLocal MP server can continue L1-only. The co-scheduled `lmcache-mp-server` is part of the required connector path for both vLLM and SGLang, so `failOpen` does not turn a missing or broken PodLocal server into a cacheless engine launch. Setting `false` makes remote storage a serving dependency and is surfaced as a Warning Event. | +| `integration.failOpen` | boolean | Default `true`. Remote L3 failure is soft by default and the MP server can continue L1-only. The PodLocal native sidecar or NodeLocal same-node server Pod is part of the required connector path, so `failOpen` does not turn a missing server into a cacheless injected engine launch. Setting `false` makes remote storage a serving dependency and is surfaced as a Warning Event. | | `integration.engineOverrides` | object | Optional engine-injection overrides applied to the args/env the pod-mutating webhook would otherwise inject into the engine container. See [Engine-injection overrides](#engine-injection-overrides-specintegrationengineoverrides). | -| `engineSelector.matchLabels` | map | Equality-based label selector matched against engine **pod** labels (the pod template's `metadata.labels`, not Deployment, DaemonSet, or any other workload-level labels). Every key/value here must appear on the pod for it to match. `matchExpressions` is intentionally not exposed in v1alpha1 — the surface is `matchLabels` only. | +| `engineSelector.matchLabels` | map | Canonical engine ownership selector matched against **pod-template** labels. A non-empty map must contain exactly one entry, `inferencecache.io/cache-domain: `, and that value must be unique among CacheBackends in the namespace. Engine Pods may carry other labels, but those labels do not participate in CacheBackend ownership. CREATE and UPDATE both enforce this shape. `matchExpressions` is not exposed. | | `hiCache` | object | Typed SGLang native HiCache configuration. Required only for `type: SGLangHiCache`; see [SGLang native HiCache](#sglang-native-hicache). | | `allowCrossNamespace` | boolean | Opt-in flag that allows `spec.remoteStorage.endpoint` to resolve to a Kubernetes Service in a different namespace from the CacheBackend itself. Without it, admission rejects cross-namespace Service-DNS endpoints. External hostnames and IPs are unaffected. Defaults to `false`. | @@ -158,7 +159,8 @@ and [`config/samples/cachebackend-external.yaml`](../../config/samples/cacheback ### Resources Current resources live with the workload owner: -`lmCache.podLocal.server.resources` for the MP native sidecar and +`lmCache.podLocal.server.resources` for the MP native sidecar, +`lmCache.nodeLocal.server.resources` for every active-node server Pod, and `remoteStorage.redis.resources` for managed Redis. The Redis renderer deep-copies the selected block onto its managed container. @@ -198,16 +200,17 @@ Limits-only shapes admit unchanged for any resource — K8s auto-populates `requ **Resource names must match K8s container-resource rules.** `ResourceList` keys are opaque map keys at the CRD-schema layer; an invalid name like `"foo"` or `""` persists in etcd and only fails when the apiserver later rejects the child pod. The validating webhook (`rejectInvalidResourceNames`) applies the same rules the apiserver applies to a `Container.Resources` map: standard names (`cpu`, `memory`, `ephemeral-storage`) admit unconditionally; a `hugepages-` name admits only when the size suffix parses as a strictly-positive `resource.Quantity` (e.g. `"hugepages-2Mi"`, `"hugepages-1Gi"` — a bare `"hugepages-"` or non-numeric `"hugepages-nope"` is rejected because the apiserver requires the size token); any other name must be **third-party vendor-prefixed** (e.g. `"nvidia.com/gpu"`) and pass `IsQualifiedName`. A bare unqualified `"foo"` is rejected even though `IsQualifiedName` alone admits it, because the apiserver's container-resource layer requires extended resources to carry a vendor identity. Names under the **K8s-reserved prefixes `kubernetes.io/` and `requests.kubernetes.io/`** are also rejected — those prefixes are reserved for native resources, so extended resources may not use them. The rejection names the offending key so multi-key errors surface together. -**Inert without a controller-managed workload.** Host-only, externally owned, -and `SGLangHiCache` configurations provision no provider Deployment or Service. -Typed PodLocal LMCache still injects its server into each matching engine Pod as -a native sidecar. HiCache host memory belongs to the user-owned engine container -and must be sized on that workload instead. +**Provider lifecycle is independent.** Host-only and externally owned LMCache +configurations provision no provider Deployment or Service. PodLocal injects a +server into each matching engine Pod; NodeLocal follows scheduled selected +engines and reconciles one direct server Pod per active node, but never creates +a load-balanced MP Service. SGLangHiCache provisions neither. HiCache host +memory belongs to the user-owned engine container. -### vLLM typed PodLocal LMCache MP support +### vLLM typed LMCache MP support The typed shape `spec.runtime: VLLM`, `spec.type: LMCache`, and -`spec.lmCache.topology: PodLocal` selects a dedicated MP adapter; it does not +`spec.lmCache.topology: PodLocal|NodeLocal` selects a dedicated MP adapter; it does not reuse the legacy `LMCacheConnectorV1` / `lm://` path. The engine image remains owned by the inference runtime. No connector-profile annotation or image allowlist is required: this CacheBackend shape is the only enablement switch. @@ -216,12 +219,16 @@ wire. The engine's normal initialization loads the connector and fails before serving if its image does not contain a compatible LMCache client/API; admission does not pull, execute, or otherwise introspect the engine image. -The webhook injects a digest-pinned `lmcache-mp-server` native sidecar and adds -the following vLLM launch contract: +For PodLocal the webhook injects a digest-pinned `lmcache-mp-server` native +sidecar. For NodeLocal it preserves engine placement, mounts host `/dev/shm`, +and adds a blocking identity/health gate while the controller follows the +scheduled engine with a same-node server Pod. Both add the following vLLM +launch contract: - `--kv-transfer-config` selects `LMCacheMPConnector` through `lmcache.integration.vllm.lmcache_mp_connector`, points it at - `tcp://127.0.0.1:`, and sets `kv_role: kv_both` for the + `tcp://127.0.0.1:` or the Downward-API-derived + `tcp://:`, and sets `kv_role: kv_both` for the only currently admitted LMCache role, `ReadWrite`; - `--disable-hybrid-kv-cache-manager` is required by the initial validated integration; @@ -236,8 +243,8 @@ because that RESP adapter cannot consume them. The initial adapter admits TP but rejects PP/DP greater than one and external multi-process DP flags. These checks and persisted webhook injection are covered without GPU; the pinned vLLM image/version, KV reuse, TP determinism, and failure recovery remain Phase -4 runtime gates. Canonical examples are the three -`config/samples/cachebackend-vllm-podlocal-*.yaml` files. +4 runtime gates. Canonical examples are the three PodLocal profiles plus +`config/samples/cachebackend-vllm-nodelocal-host-only.yaml`. For both typed vLLM and SGLang PodLocal adapters, `l1Capacity` is the usable L1 target, not the complete container budget. The common renderer creates a @@ -248,6 +255,46 @@ memory-backed `emptyDir` with a `sizeLimit` at least as large as that budget. This keeps scheduling/cgroup accounting aligned with the tmpfs and leaves room for LMCache metadata and shared-memory allocator overhead. +For NodeLocal, `l1Capacity` is instead one shared per-node budget. CacheBackend +creation alone creates no server and never changes engine placement. After an +injected engine has been scheduled, the controller owns one host-networked +server Pod for each distinct active engine node, declares the MP and FastAPI +listeners as host ports, and mounts the node's `/dev/shm` into both the server +and selected engine Pods. Exact node-name affinity sends the server through the +normal scheduler on the engine's node; `status.hostIP` prevents ClusterIP or +cross-node CUDA IPC. `maxGPUWorkers` must cover all selected engine instances on +one node. Every server receives the controller-derived +`lmcache_l1_pool_inferencecache_` through `--shm-name`; the +engine gate verifies both the declared MP value and the effective L1 +memory-manager value before startup. Different CacheBackend UIDs therefore do +not accidentally unlink or rebind the same POSIX SHM object. The server sets +`NVIDIA_VISIBLE_DEVICES=all` but requests no +allocatable GPU. It inherits the source engine's runtime class, tolerations, +image-pull secrets, priority class, and scheduler unless optional +`nodeLocal.scheduling` server overrides are supplied. The FastAPI/MP listeners +are unauthenticated and host networking bypasses NetworkPolicy, so this topology +requires one trusted tenant domain per pool plus node firewall controls. +CacheBackend name/UID/generation verification detects wrong ownership but is +not cryptographic authentication, and unique names do not isolate hostile +processes that already have compatible access to host `/dev/shm`. Co-located +pools therefore remain limited to one trusted node domain. After the last +selected engine leaves a +node, `nodeLocal.idleRetentionSeconds` keeps the server and shared L1 warm for +the configured window (300 seconds by default); new demand on that node reuses +the same Pod. Set it to zero for immediate deletion. A retained Pod continues +to reserve its declared host ports, so another NodeLocal backend using the same +pair remains in the normal Kubernetes host-port conflict path until expiry. + +NodeLocal ports are explicit rather than dynamically allocated. Engine Pods +are immutable and receive their endpoint during admission, before the +on-demand server exists; Kubernetes dynamically allocates Service node ports, +not direct Pod host ports, and a Service is not a valid CUDA MP endpoint. A +CacheBackend also declares one runtime, so vLLM and SGLang never share one +server pool. The selector must describe one runtime/model/cache-layout and +trust domain; different prompts within that domain are separated by LMCache KV +keys, while a different model, runtime, package baseline, or tenant needs a +separate backend with disjoint ports on shared nodes. + ### SGLang engine support SGLang supports two peer cache integrations: @@ -256,6 +303,7 @@ SGLang supports two peer cache integrations: |---|---|---| | `(SGLang, LMCache)` without `remoteStorage` | PodLocal LMCache MP server, host-only | Native sidecar in each selected engine Pod | | `(SGLang, LMCache)` with Managed Redis | PodLocal LMCache MP server with a shared Redis remote tier | Native sidecar plus Redis Deployment and Service | +| `(SGLang, LMCache)` with `topology: NodeLocal` | Same-node shared LMCache MP server | One CacheBackend-owned server Pod per active engine node; optional Redis remains independent | | `(sglang, SGLangHiCache)` | Native engine-local host cache | None | #### SGLang LMCache MP mode @@ -317,12 +365,17 @@ The old lm:// `LMCACHE_REMOTE_URL` / serde / chunk-size / local-CPU env is | Field | Default | Bounds | Purpose | |---|---|---|---| | `lmCache.chunkSizeTokens` | `256` | `>=1` | Server chunk size and client config. | -| `lmCache.topology` | required | `PodLocal` | `NodeLocal` is a future shape and is rejected. | +| `lmCache.topology` | required | `PodLocal`, `NodeLocal` | Chooses native-sidecar or engine-demanded per-node server placement. | | `lmCache.podLocal.server.image` | required | digest-pinned reference | Independently owned LMCache server image; never copied from or into the engine image. | | `lmCache.podLocal.server.port` | required | `1`–`65535` | Loopback MP port. | | `lmCache.podLocal.server.l1Capacity` | required | positive quantity | Usable L1; `/dev/shm` and memory resources must cover this plus 1Gi. | | `lmCache.podLocal.server.maxWorkers` | required | `>=1` | Server worker bound. | | `lmCache.podLocal.server.resources` | required | validated K8s resources | Positive CPU request and sufficient memory request/limit. | +| `lmCache.nodeLocal.server.{image,port,httpPort}` | required | digest plus distinct ports | One server image/config and real node-bound listeners for the pool. | +| `lmCache.nodeLocal.server.l1Capacity` | required | positive quantity | Shared L1 budget per active engine node; memory request/limit cover it plus 1Gi. | +| `lmCache.nodeLocal.server.{maxGPUWorkers,maxCPUWorkers}` | required | `>=1` | Shared per-server worker bounds. | +| `lmCache.nodeLocal.idleRetentionSeconds` | `300` | `0`–`86400` | Warm retention after the last selected engine leaves a node; `0` deletes immediately. | +| `lmCache.nodeLocal.scheduling` | optional | server operational overrides | May override tolerations, image-pull secrets, ServiceAccount, Pod security context, priority/scheduler, runtime class, and termination grace on server Pods. It does not expose node selection or mutate engine placement. | Deliberately **not** injected for SGLang (a real engine difference, not an omission): `VLLM_USE_V1` (a vLLM-internal codepath with no SGLang analogue) and `PYTHONHASHSEED` (vLLM pins it to stabilise its builtin-`hash()`-seeded block-hash chain across TP workers; SGLang derives its prefix hash with `hashlib.sha256` over the token-id bytes, independent of `PYTHONHASHSEED`). @@ -357,7 +410,7 @@ spec: type: SGLangHiCache engineSelector: matchLabels: - app: sglang + inferencecache.io/cache-domain: sglang-hicache hiCache: # Exactly one: ratio: "2.0" @@ -625,9 +678,9 @@ The poller attributes each `/snapshot.replicas[]` entry to a single owning `Cach 1. Looks up the engine pod by `(tenant, replicaID)`. 2. If the pod carries the webhook's `inferencecache.io/injected-by` annotation (stamped as `/`), resolves the owning CacheBackend directly. This is the authoritative wiring signal — the engine container was wired to exactly that backend's endpoint. -3. Otherwise, iterates that namespace's CacheBackends sorted by `metadata.name` and picks the first whose `spec.engineSelector.matchLabels` is non-empty and is a subset of the pod's labels. This mirrors the pod webhook's first-match rule for pods that bypassed the webhook (manual sidecar attachment, opt-out). +3. Otherwise, iterates that namespace's CacheBackends sorted by `metadata.name` and picks the first whose `spec.engineSelector.matchLabels` is non-empty and is a subset of the pod's labels. This fallback exists only for manually attached subscriber Pods that bypassed normal webhook injection. CacheBackend admission rejects duplicate ownership, and the Pod webhook rejects a fresh Pod with multiple matches rather than using this attribution fallback to choose its connector owner. -Only ONE CacheBackend ever claims a given replica — overlapping selectors must agree on which backend owns the pod, otherwise status would disagree with what the engine was actually wired to. A CacheBackend without an EngineSelector (or with empty `MatchLabels`) is excluded from the selector fallback — otherwise a misconfigured backend would silently claim every replica in its namespace by vacuous truth — but a pod can still be attributed to it via the `injected-by` annotation. A replica whose pod can no longer be found (drained between events and now) is skipped; its data still appears in the cluster-wide `CacheIndex`. A failing scrape preserves existing state (soft-state); a successful scrape that finds no matching replicas resets `prefixCount` to `0` so stale positive values do not survive a drain. +Every newly admitted engine has exactly one CacheBackend owner. Every non-empty selector contains only the namespace-unique `inferencecache.io/cache-domain` label. The Pod webhook denies runtime ambiguity caused by concurrent CREATE races. A CacheBackend without an EngineSelector (or with empty `MatchLabels`) is excluded from the selector fallback — otherwise a misconfigured backend would silently claim every replica in its namespace by vacuous truth — but a pod can still be attributed to it via the `injected-by` annotation. A replica whose pod can no longer be found (drained between events and now) is skipped; its data still appears in the cluster-wide `CacheIndex`. A failing scrape preserves existing state (soft-state); a successful scrape that finds no matching replicas resets `prefixCount` to `0` so stale positive values do not survive a drain. ## Contract Notes @@ -662,8 +715,8 @@ autoscaling, provider images, or an engine image. Validation aggregates field-scoped violations into one Kubernetes `Invalid` response. The current rules enforce: -- a typed LMCache `PodLocal` topology (NodeLocal is published but rejected - until Phase 8), a digest-pinned MP-server image, non-colliding ports, and +- a typed LMCache `PodLocal` or `NodeLocal` topology, a digest-pinned MP-server + image, non-colliding ports, explicit NodeLocal placement, and sufficient CPU/memory resources; - Redis as the only remote provider, with explicit Managed/External ownership, a valid External endpoint, provider/config agreement, and only RESP features @@ -715,8 +768,8 @@ rejects any override or suppression that overlaps those lists: | Adapter | Reserved args | Reserved env | |---|---|---| -| vLLM typed PodLocal MP | `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager` | `PYTHONHASHSEED`, `INFERENCECACHE_FAIL_OPEN` | -| SGLang typed PodLocal MP | `--enable-lmcache`, `--lmcache-config-file`, `--enable-metrics` | `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN` | +| vLLM typed MP | `--kv-transfer-config`, `--disable-hybrid-kv-cache-manager` | `PYTHONHASHSEED`, `INFERENCECACHE_FAIL_OPEN` | +| SGLang typed MP | `--enable-lmcache`, `--lmcache-config-file`, `--enable-metrics` | `LMCACHE_USE_EXPERIMENTAL`, `INFERENCECACHE_FAIL_OPEN` | | SGLangHiCache | its injected HiCache flags | none unless introduced by the adapter | The removed IP connector environment is neither injected nor part of the @@ -748,10 +801,10 @@ A separate mutating admission webhook on `corev1/v1.Pod` (`name: mpod.inferencec | Aspect | Behavior | |---|---| -| Selection | Lists `CacheBackend`s in the pod's namespace via the manager's **APIReader** (uncached live client; an informer-cache miss on a freshly-Ready backend would leave the pod permanently unwired since pod CREATE is a one-shot), then matches `pod.Labels` against each `Spec.EngineSelector.MatchLabels`. The first matching `CacheBackend` wins; one with a nil or empty `EngineSelector` is skipped (a "match-everything" selector would silently claim every pod in the namespace). | +| Selection | Lists `CacheBackend`s in the pod's namespace via the manager's **APIReader** (uncached live client; an informer-cache miss on a freshly-Ready backend would leave the pod permanently unwired since pod CREATE is a one-shot), then matches `pod.Labels` against each `Spec.EngineSelector.MatchLabels`. Exactly one match is required. Zero matches pass through unmodified; multiple matches deny Pod admission and name every conflicting backend. CacheBackend admission normally prevents this shape; the Pod check closes the concurrent-CREATE race. A nil or empty `EngineSelector` is skipped. | | Injection | Resolves the runtime adapter via `runtime.Registry.Select(runtimeID, cache)`, resolves `spec.remoteStorage` independently, and constructs a structured provider `Binding{Protocol, Endpoint}`. Managed ownership uses `status.remoteStorage.endpoint` from the live Service; External ownership uses the trimmed, provider-validated `spec.remoteStorage.endpoint` with no fallback to stale status; omitted `remoteStorage` produces a nil host-only binding. `SupportsBinding` is part of the required runtime adapter interface, and the webhook passes the binding directly to `adapter.InjectEngineConfig`, so the adapter selects host-only MP or the RESP wire from the binding protocol instead of inferring storage from `spec.type`. A non-nil binding with a missing endpoint fails open. Events-only skips engine injection because it wires no KV connector and appends only the kvevent-subscriber sidecar. Adapters preserve existing user args/env and make repeat injection idempotent. | | Annotations | Stamps TWO annotations on every successfully mutated pod: `inferencecache.io/injected-by: /` (operator-readable identity, shows in `kubectl describe pod`) AND `inferencecache.io/injected-by-uid: ` (the matched CR's metadata.uid). Successful injection also clears any stale `inferencecache.io/inject-skipped` marker. Reads `inferencecache.io/skip-inject: ` as an opt-out: the webhook returns Allowed, skips engine wiring, clears any stale injected-by/injected-by-uid pair, and stamps `inferencecache.io/inject-skipped: skip-inject-annotation` so explicit operator opt-out is distinguishable from selector drift. On all other fail-open returns after the pod is decoded (list/no match/missing endpoint/adapter errors), the webhook strips stale injected-by/injected-by-uid and inject-skipped annotations so a user cannot trick the events controller by pre-stamping a pod template. Decode failures fail open before a Pod exists to patch, so stale annotations cannot be cleared on that path. | | Events | The webhook itself does NOT record events (the apiserver assigns `metadata.uid` after mutating admission, so a webhook-recorded event would carry `involvedObject.uid=""` and be invisible to `kubectl describe pod`). Instead, the pod-watching `engine-pod-events` controller reads the persisted decision annotations after CREATE. For injected pods, it validates `inferencecache.io/injected-by-uid` against the live CR's `metadata.uid` and records a `Normal InjectedByCacheBackend` event on the now-persisted pod. For explicitly skipped pods carrying both a truthy `inferencecache.io/skip-inject` and `inferencecache.io/inject-skipped: skip-inject-annotation`, it records a `Normal SkippedByOperator` event on that pod. The skip marker is not authenticated, and `skipInjection` treats a pre-existing correct marker as already converged; `SkippedByOperator` therefore means the persisted pod carries the explicit opt-out plus skipped marker, not proof that the webhook authored the marker. The UID match REDUCES — but does NOT eliminate — the failurePolicy=Ignore forgery surface for injected pods: a casual copy-paste of an injected pod's annotations into a fresh template won't match the live CR's UID, but `metadata.uid` is not secret, so a pod creator with `get` RBAC on CacheBackends can read it and stamp the pair correctly. The injected Event signals "the webhook claims this pod was injected and the claim is consistent with the live CR," not "the webhook was cryptographically authenticated." The controller skips the injected event when the CR is missing, the UID annotation is absent, or the UID does not match — see the controller godoc for the full skip table. controller-runtime's EventBroadcaster aggregates duplicates on the apiserver side, so a re-enqueue across controller restarts upserts the existing event rather than spamming. | | Idempotency | The handler calls the adapter unconditionally on every admission and trusts the adapter to converge the full injected contract. For LMCache this is env plus the engine-specific required surface — `--kv-transfer-config` for vLLM; for SGLang `--enable-lmcache` + `--lmcache-config-file` **plus** the MP-worker native sidecar and the shared config / `/dev/shm` volumes + mounts. Its merge primitives (`upsertEnv` / `upsertArgPair` / `upsertFlag`, and for SGLang `adoptContainer` / `adoptVolume` / `upsertMountByName`) converge on the desired value rather than appending a duplicate. The SGLang `adopt*` pair additionally distinguishes the adapter's own prior injection (converge) from an operator's object squatting a reserved name (reject → fail-open admit) — see [Names the MP wire reserves](#sglang-engine-support). Native HiCache validates all reserved arguments against the original pod before mutation, preserves one matching or well-formed operator-supplied value, appends each missing canonical argument once, and rejects conflicts, malformed values, or duplicates without partially changing the pod. Re-admission of a fully-injected pod therefore produces an empty JSON-patch set. Trusting the adapter rather than a handler-side env-presence shortcut avoids the trap where a partially-injected pod is admitted permanently missing the rest of the contract. | -| Fail-open | Every error path (decode failure, list error, no matching backend, missing managed `status.remoteStorage.endpoint`, no registered adapter, adapter rejection, re-encode failure) returns `admission.Allowed(...)` with a reason — webhook errors MUST NOT block engine admission. `MutatingWebhookConfiguration.failurePolicy` is also pinned to `Ignore` as a belt-and-suspenders second layer. | +| Fail-open | Operational paths (decode/list errors, no matching backend, missing managed `status.remoteStorage.endpoint`, no registered adapter, adapter rejection, re-encode failure) return `admission.Allowed(...)` with a reason. Selector ambiguity is the intentional exception: a live webhook denies the Pod rather than choosing an unintended cache trust domain. `MutatingWebhookConfiguration.failurePolicy=Ignore` still protects engine availability during webhook transport outages; the CacheBackend validating webhook has `failurePolicy=Fail` and rejects overlapping selectors in the normal path. | | Verbs | `CREATE` only. UPDATE re-admissions to a running pod don't re-inject (and the engine container can't pick up env changes without a restart anyway); UPDATEs to engine pods are rare in this fleet. | diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 764fb385..8c866e91 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -1,6 +1,6 @@ # Design Roadmap: LMCache Multiprocess Migration -Status: **Phases 0–5 and 7 complete; Phase 6 was not required (2026-08-11)** · Scope: +Status: **Phases 0–5, 7, and 8 complete; Phase 6 was not required (2026-08-12)** · Scope: deprecate and remove this project's LMCache in-process data plane, converge vLLM and SGLang on LMCache multiprocess (MP) mode, and model Pod-local and node-local MP server placement without conflating either with @@ -73,8 +73,9 @@ type. In particular: connector. Existing code and documents sometimes call it an MP worker. - **PodLocal** means one MP server native sidecar per engine Pod, reached over loopback and sharing that Pod's `/dev/shm`. -- **NodeLocal** means one MP server Pod per node, normally a DaemonSet member, - shared by engine Pods on that node. +- **NodeLocal** means one on-demand MP server Pod per node that currently hosts + selected engine Pods. The inference system schedules engines first; the + cache controller follows that observed placement. - **Remote storage** means only the optional L3 behind the IP connector or MP server. An MP server is never declared as `remoteStorage`. - **Legacy LMCacheServer** means the legacy IP centralized-sharing service @@ -95,7 +96,7 @@ code lands. | D2 | `remoteStorage` is optional L3 only. | Local CPU capacity and MP server placement are engine-integration concerns, not remote-provider selection. | | D3 | `LMCacheServer` is removed from the canonical `remoteStorage.provider` set. | `lm://` is a legacy IP remote connector and is absent from the MP L3 adapter catalog. | | D4 | PodLocal is the first production candidate and migration target. | It has the smallest scheduling and ownership surface and builds on the existing SGLang proof. | -| D5 | NodeLocal means a per-`CacheBackend` DaemonSet in its first implementation. | Multiple engine Pods of one backend may share it; cross-`CacheBackend` sharing introduces unresolved config, tenancy, port, and deletion ownership. | +| D5 | NodeLocal means one controller-owned server Pod per active engine node per `CacheBackend`; the inference system remains the placement authority. This replaces the earlier DaemonSet/server-first decision on 2026-08-12. | A DaemonSet requires a node set before engines are scheduled and therefore inverted ownership by forcing engines onto cache-selected nodes. Engine-demanded Pods preserve arbitrary inference-system scheduling while still allowing same-node sharing. | | D6 | A generic Deployment behind a load-balanced Service is not a valid CUDA MP topology. | CUDA IPC and shared memory require the engine to reach the MP server on its own node. | | D7 | Connector endpoints are not published in the generic `status.endpoint`. | PodLocal uses loopback; NodeLocal is node-dependent. Only remote L3 has a globally meaningful provider endpoint. | | D8 | Unsupported combinations are rejected at admission. | An accepted but inert cache field commonly produces silent zero-hit behavior. | @@ -103,6 +104,7 @@ code lands. | D10 | Component lifecycle ownership is capability-specific. | The Pod-local MP process is kubelet-owned while remote L3 is independently managed; connector re-registration after an MP-process restart is a post-migration enhancement, not an MVP contract. | | D11 | Each supported vLLM integration explicitly identifies its MP connector implementation; the initial reference baseline uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial adapter uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future adapter revision may validate a different implementation explicitly. | | D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. The selected adapter renders its engine-specific connector contract, while normal engine initialization is the authoritative compatibility check; tested images are neither an admission allowlist nor a mutation default. | +| D13 | Selecting `NodeLocal` explicitly opts the backend into one host-networked MP server per active engine node and host `/dev/shm` mounts in both server and selected engine Pods; engine Pods themselves remain off host networking and host IPC. | LMCache 0.5.3 requires node-visible networking and shared host memory for cross-Pod CUDA IPC. Keeping engine placement and networking under the inference system reduces coupling, while the topology choice and documented trust domain make the remaining host access explicit. | ## Migration baseline (before Phase 1) @@ -187,7 +189,7 @@ GPU node +-----------+-----------+ v +--------------------+ - | LMCache MP server | one DaemonSet Pod per node + | LMCache MP server | one on-demand Pod per active node | shared CPU L2 | +---------+----------+ | @@ -197,13 +199,15 @@ GPU node Properties: -- one MP server per eligible node per `CacheBackend`; +- one MP server per node currently hosting selected engines for a + `CacheBackend`; - multiple selected engine Pods on that node share CPU cache capacity; - engines derive the endpoint from their node identity, not a load-balanced Service endpoint; - L2 capacity is per node; -- server scheduling, host port, host shared-memory arrangement, GPU visibility, - and node coverage become controller-owned concerns; +- engine scheduling remains inference-system-owned; server exact-node binding, + host port, host shared-memory arrangement, GPU visibility, and node coverage + become controller-owned concerns; - cross-`CacheBackend` and cross-tenant sharing are out of scope for the first implementation. @@ -244,7 +248,7 @@ spec: failOpen: true engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-lmcache ``` Omit `remoteStorage` for host-only MP operation. @@ -265,6 +269,7 @@ spec: server: image: registry.example/lmcache-standalone@sha256:... port: 6555 + httpPort: 8080 l1Capacity: 128Gi maxGPUWorkers: 8 maxCPUWorkers: 8 @@ -275,16 +280,14 @@ spec: limits: memory: 132Gi scheduling: - nodeSelector: - inferencecache.io/lmcache-mp: "true" - tolerations: [] + runtimeClassName: nvidia # optional server override; never selects nodes remoteStorage: provider: Redis ownership: External endpoint: redis.example:6379 engineSelector: matchLabels: - app.kubernetes.io/name: vllm + inferencecache.io/cache-domain: vllm-lmcache ``` ### Final provider matrix @@ -293,8 +296,8 @@ spec: |---|---|---:|---:|---:|---:| | SGLang | PodLocal | required MVP | required MVP | future | rejected | | vLLM | PodLocal | required MVP | required MVP | future | rejected | -| SGLang | NodeLocal | planned | planned | future | rejected | -| vLLM | NodeLocal | planned | planned | future | rejected | +| SGLang | NodeLocal | implemented; GPU pending | functional GPU passed; metrics pending | future | rejected | +| vLLM | NodeLocal | implemented; functional GPU passed | functional GPU passed; metrics pending | future | rejected | “Required MVP” means the combination must be implemented and validated, not that the remote L3 field itself is required. @@ -316,6 +319,12 @@ status: readyServers: 4 coveredEnginePods: 8 uncoveredEnginePods: 0 + enginePodCoverage: + - name: engine-0 + nodeName: gpu-node-a + ready: true + covered: true + reason: ConnectorReady remoteStorage: provider: Redis endpoint: redis.example:6379 @@ -332,9 +341,9 @@ status: Required semantics: - PodLocal `desiredServers` equals the selected engine Pod count. -- NodeLocal `desiredServers` equals the number of distinct nodes hosting selected - engine Pods, or the explicitly managed eligible-node count when the DaemonSet - is intentionally prewarmed. +- NodeLocal `desiredServers` equals the number of distinct `spec.nodeName` + values among active engine Pods carrying this CacheBackend's valid name+UID + injection record. Unscheduled engines demand no speculative server yet. - `coveredEnginePods` counts selected engine Pods whose required MP server is healthy and reachable. - PodLocal loopback and NodeLocal node-derived connector addresses are not @@ -366,8 +375,8 @@ the legacy deprecation writer is only implemented if Phase 6 is activated. | 4 | vLLM PodLocal MP | Phase 3 | complete | | 5 | Repository consumer migration; migration tooling only if needed | Phase 4 | complete | | 6 | Conditional compatibility gate if legacy consumers appear | Phase 5 | not required by Phase 0 and Phase 5 findings | -| 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | in progress | -| 8 | NodeLocal shared MP server topology | Phases 3–4; does not block Phase 7 | not started | +| 7 | Remove IP, `lm://`, and LMCacheServer provider | Phase 5; Phase 6 only when applicable | complete | +| 8 | NodeLocal shared MP server topology | Phases 3–4; does not block Phase 7 | complete | ## Phase 0 — design freeze and compatibility baseline @@ -869,7 +878,8 @@ skipped because `xxhash` was unavailable; `make ci` still passed. ## Phase 8 — NodeLocal shared MP servers -- **Status:** Not started +- **Status:** Complete (2026-08-12), including focused live-node and GPU + validation of the UID-scoped POSIX SHM remediation. - **Depends on:** Phases 3–4; does not block Phase 7 ### Objective @@ -879,131 +889,287 @@ node without weakening placement, isolation, or status correctness. ### Scope -Includes same-node discovery, DaemonSet lifecycle, shared capacity, and -multi-node coverage. Cross-`CacheBackend` sharing remains out of scope. +Includes engine-first same-node discovery, on-demand per-node server lifecycle, +shared capacity, multi-node coverage, host-port conflict handling, and status. +PodLocal remains supported. SGLang TP>1, multi-node TP, distributed executors, +MLA, directional PD roles, typed MP Mooncake L2, managed Redis clustering, +generic MP-server restart/re-registration, and cross-`CacheBackend` server +sharing remain out of scope. + +The final contract is: + +- **API:** `nodeLocal.server` explicitly requires a digest-pinned image, + distinct MP and FastAPI host ports, per-node L1 capacity, GPU/CPU worker + limits, and resources covering `l1Capacity + 1Gi`. The FastAPI listener also + serves `/metrics`; no third metrics port is created. `nodeLocal.scheduling` + exposes only server operational overrides and cannot select nodes. + `nodeLocal.idleRetentionSeconds` defaults to 300, accepts 0–86400, and owns + warm server/L1 retention independently from engine-Pod lifetime. Host-bound + and security-relevant values have no implicit defaults. +- **Lifecycle and placement:** CacheBackend creation alone creates no MP server. + The inference system schedules engines first; the controller then owns one + direct server Pod per distinct active engine node. Required node affinity to + `engine.spec.nodeName` preserves engine placement while keeping normal + host-port, taint, resource, and scheduler checks active. After the last + selected engine leaves a node, typed `idleRetentionSeconds` retains that + server and L1 for reuse; expiry removes it, while zero requests immediate + deletion. No Deployment, ReplicaSet, or DaemonSet owns these Pods. +- **Host boundary:** Server Pods use `hostNetwork`, `ClusterFirstWithHostNet`, + host `/dev/shm`, the selected NVIDIA runtime without reserving allocatable + GPUs, and a restrictive container security context. Engine Pods remain off + host networking and host IPC. +- **Endpoint and gate:** Engines derive the same-node address from Downward API + `status.hostIP`. A blocking init gate requires healthy `/healthcheck` plus an + exact `/config` match for namespace/name/UID/generation, ports, and chunk size + before the engine starts. It also verifies both the declared MP `shm_name` + and the effective L1 memory-manager `shm_name`, so an unsafe pool or LMCache + shared-memory fallback cannot silently admit the engine. SGLang writes its + engine-specific client YAML; vLLM retains its connector JSON. No Service or + ClusterIP participates in CUDA MP traffic. +- **Ownership and isolation:** One CacheBackend name/UID/runtime and its sole + namespace-unique `inferencecache.io/cache-domain` value own one server pool. + CREATE and UPDATE reject non-canonical or duplicate ownership; Pod admission + denies concurrent ambiguity. Every server receives the full UID-derived + `lmcache_l1_pool_inferencecache_` name through `--shm-name`; the name is + stable across same-UID generation/server replacement and distinct after + CacheBackend delete/recreate. Disjoint port pairs prevent network bind + conflicts, while UID-scoped names prevent accidental POSIX SHM unlink/rebind + between co-located pools. Idle-retained servers continue reserving their + ports and SHM budget until expiry. UID matching and a unique SHM name are + routing/ownership identities, not authentication: co-located pools must + remain inside one mutually trusted node domain, host firewall controls are + required, and NetworkPolicy does not isolate host-network listeners. +- **Runtime consistency:** A pool cannot mix vLLM and SGLang. CacheBackend + supplies one server image, chunk size, port tuple, generation, and runtime for + the pool. The inference-system owner remains responsible for engine + image/package/model compatibility; normal engine initialization is + authoritative. Different prompts within one compatibility domain may share + content-addressed KV entries, while another runtime/model/layout/tenant + domain requires a separate CacheBackend. +- **Status:** `desiredServers` is the distinct active scheduled-engine node + count. `readyServers` counts current-generation, name/UID-verified Ready + servers carrying the expected UID-scoped SHM annotation and exact + `--shm-name` argument on those nodes. An engine is covered only by exactly one + healthy current server on its own node; unscheduled, stale-generation, + SHM-mismatched, ambiguous, or serverless engines are uncovered. Connector + readiness requires all desired servers and all matched engines to be Ready + and covered. +- **Capacity:** `l1Capacity` is one shared budget per active node, not per engine + Pod. `maxGPUWorkers` must cover the maximum engine instances expected on one + node. ### Deliverables -- [ ] Reconcile one DaemonSet per NodeLocal `CacheBackend`. -- [ ] Restrict it to intended GPU/engine nodes through typed scheduling fields. -- [ ] Configure host networking and host shared memory according to the pinned - upstream deployment contract. -- [ ] Declare host ports so Kubernetes scheduling exposes conflicts. -- [ ] Authenticate ownership by CacheBackend name and UID. -- [ ] Compute desired/ready servers and engine-node coverage. -- [ ] Handle engine scheduling before the node-local server is ready without - starting an engine against a missing required MP endpoint. -- [ ] Derive the node-local address from the engine Pod's node/host IP through a - Downward API field or another deterministic node-scoped mechanism. -- [ ] Do not use a load-balanced ClusterIP as the CUDA MP endpoint. -- [ ] Keep SGLang and vLLM launch surfaces engine-specific. -- [ ] Validate the server's global chunk size and version against every selected - engine Pod. -- [ ] Define port-conflict behavior for multiple NodeLocal CacheBackends on one - node. -- [ ] Restrict the first implementation to one trust/tenant domain per - CacheBackend server pool. -- [ ] Document that L1 capacity is per node and shared by selected engine Pods. -- [ ] Size `maxGPUWorkers` for the number of engine instances sharing a server. -- [ ] Add NetworkPolicy/firewall guidance where host networking permits it. -- [ ] Assess the security impact of host networking/shared memory and GPU - visibility. +- [x] Reconcile one controller-owned server Pod per distinct active scheduled + engine node, retain it for the typed idle window after final demand, and + delete it on expiry. +- [x] Preserve inference-system engine placement and bind the server through + exact-node affinity so Kubernetes still checks ports and resources. +- [x] Configure the required host-network, host-shared-memory, GPU-visibility, + probe, resource, and restrictive security surfaces. +- [x] Declare both host ports and surface same-node conflicts without accepting + another backend's listener. +- [x] Require one namespace-unique `inferencecache.io/cache-domain` selector on + CREATE and UPDATE; deny ambiguous Pod injection and cross-backend sharing. +- [x] Gate engine startup on the healthy same-node server's exact + name/UID/generation/port/chunk-size identity. +- [x] Derive one full UID-scoped POSIX SHM name per CacheBackend, pass it + explicitly to every NodeLocal server, verify declared and effective live + configuration, and replace or un-cover servers missing that identity. +- [x] Derive the endpoint from Downward API node data and render no MP Service + or load-balanced ClusterIP. +- [x] Keep vLLM and SGLang launch/configuration surfaces separate while sharing + the common server renderer. +- [x] Compute desired/ready servers and per-engine same-node coverage. +- [x] Enforce one trust/tenant/runtime/model/layout domain per server pool and + leave engine package compatibility to normal engine initialization. +- [x] Define L1 as a per-node shared budget and require `maxGPUWorkers` sizing + for all engines expected on that node. +- [x] Document the host-network, shared-memory, GPU, firewall, isolation, and + failure-domain boundaries. ### Validation -- [ ] One engine Pod on one node. -- [ ] Multiple engine Pods sharing one node-local server. -- [ ] Engines spread across multiple nodes, each using only its local server. -- [ ] Node drain and engine rescheduling. -- [ ] Host-port conflict negative test. -- [ ] Redis outage/recovery with multiple node-local servers. -- [ ] No cross-node attempt to use CUDA IPC. +An initial DaemonSet/server-first prototype was tested and then rejected during +architecture review on 2026-08-12 because it made CacheBackend placement +authoritative over the inference system. None of that prototype's results count +toward Phase 8. The evidence below is for the replacement engine-first, +on-demand server-Pod implementation only. + +Local and repository validation completed on 2026-08-12 PDT: + +| Check | Result | +|---|---| +| Generated API artifacts | `make generate manifests` passed after the final engine-first/idle-retention API change; deepcopy, served CRD, Pod create/patch/delete RBAC, and webhook manifests are synchronized. No DaemonSet RBAC remains. | +| Unit/envtest | `git diff --check` and `go test ./...` passed. Tests cover zero-server-without-engine, one server per distinct scheduled node, multiple same-node engines, idle marking/reuse/expiry and zero-retention cleanup, generation replacement, foreign-name collision, cross-CacheBackend ownership rejection, exact-node affinity, Pod watch mapping, same-node status coverage, optional scheduling overrides, strict canonical one-label cache-domain validation on CREATE and UPDATE, duplicate-domain rejection, ambiguous-Pod denial, doctor ambiguity reporting, and vLLM/SGLang placement-preserving injection. The real Kubernetes 1.31 admission envtest also passed an API-server CREATE rejection for a duplicate cache domain. | +| Samples | `make verify-samples` passed: 27 admitted, one pre-existing explicit skip, zero failures. Both engine-first NodeLocal samples passed real admission. | +| Coverage | `make cover-check` passed. | +| CI | The baseline was amended as `628194e` with a matching `Signed-off-by`; `make verify-dco` and the complete `make ci` target passed. The optional Python golden-vector regeneration explicitly skipped because `xxhash` is unavailable. | +| Fresh install | Dedicated Kubernetes 1.32 kind clusters passed the engine-first CRD/controller/webhook installation, real duplicate cache-domain rejection, zero speculative server Pods, placement-preserving NodeLocal engine admission, scheduler-selected engine node followed by one exact-node-affinity direct server Pod, host boundary/host ports, no MP Service, PodLocal admission, current samples, doctor, idempotent re-apply, served idle-retention default/bounds, Pod patch RBAC, idle marking, and same-UID reuse. VPN-safe temporary self-signed webhook TLS replaced only the cert-manager download; both clusters were deleted. The later UID-scoped `--shm-name` delta updated this smoke with exact annotation/argument checks but was not rerun end-to-end because no kind node image/cluster was cached and the standard cert-manager URL remained unavailable through the VPN. That delta instead passed real envtest admission plus the SJC current-controller live tests recorded below. | +| Legacy production search | Production Go/manifests contain no `LMCacheConnectorV1`, `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, `ProtocolLMCache`, `lm://`, or LMCacheServer provider path. Remaining LMCacheServer matches are sample comments explicitly describing its removal. | +| Confirmed SHM collision root cause | A focused SJC dev test placed two independent LMCache 0.5.3 standalone Pods on node `10.0.103.182`, with disjoint ports and instance IDs but no `--shm-name`. Both ran as PID 1: A created `/dev/shm/lmcache_l1_pool_1` at inode `14092`; B unlinked that name and recreated inode `14097`; A retained a mapping to deleted inode `14092`. The dedicated namespace was deleted and no control-plane object changed. | +| UID-scoped SHM implementation | `git diff --check`, gate-script Python syntax parsing, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check`, and complete `make ci` passed. Tests cover deterministic full-UID naming, distinct UIDs, unsafe/oversized UID rejection, exact server args/annotation, declared and effective startup-gate checks, status exclusion, automatic replacement of an existing server missing the managed SHM identity, and PodLocal regression. Fresh-install could not run locally because no kind node image/cluster is cached and the standard cert-manager bootstrap requires the known-unavailable GitHub path; real envtest API-server admission did run. | +| Focused live SHM remediation | On SJC Kubernetes 1.31.1, two raw LMCache 0.5.3 servers with distinct explicit UID-style names ran together on CPU node `10.0.103.182`: A remained at inode `14166`, while B used inode `14176` and then `14181` after replacement. A's mapping remained named and unchanged throughout. The current controller then created two independent NodeLocal pools on the same node with real CacheBackend UIDs `61a98028-653a-4cfb-83ef-2dc3a9321b50` and `47205c02-6d7d-45aa-bf40-0e1882346309`: their effective names and inodes were respectively `14196` and `14200`; replacing only B moved it to `14207` while A stayed `14196`; deleting and recreating B's engine demand inside idle retention reused B's same server Pod UID and inode `14207`. Both pools reported server/engine coverage `1/1/1/1`. All CPU test resources were deleted. | + +SJC engine-first GPU validation ran on 2026-08-12 PDT: + +The dedicated `inference-cache-gpu-test` namespace ran Kubernetes 1.31.1 on +two BM.GPU.A100-v2.8 nodes with A100-SXM4-80GB GPUs, NVIDIA driver 550.163.01, +and CUDA 12.9 client artifacts. The server image was +`docker.io/lmcache/standalone@sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b`. +The vLLM 0.25.1 engine image was +`sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a`; +the SGLang 0.5.13.post1 engine image was +`sha256:920df39109c60429b0a23eaacfd2786fcf1595c12f3ca4fc6e153b2abe34865f`. +Both engine manifests used the LMCache 0.5.3 CUDA 12.9 wheel with SHA-256 +`3587d26a23e942b774589c88ea4cfb019af53474aba84dc772dae5900f9ad2cb` +as test-only runtime-owner scaffolding. It is not a production installation +mechanism and was not supplied by CacheBackend. + +| Check | Evidence/result | +|---|---| +| Engine-first lifecycle and gating | Each CacheBackend initially created only its managed Redis Deployment/Service and zero MP servers. After the inference-system-owned engine Pod was scheduled, the controller created the same-node direct server Pod. All gates first observed connection refusal, then admitted the engine only after `/config` and `/healthcheck` matched the exact CacheBackend namespace/name/UID/generation and typed port/worker/chunk configuration. The test Pods had no application readiness probe, so data-plane testing additionally waited for the engine's `Application startup complete`; Kubernetes Pod Ready alone was not treated as service readiness. | +| Effective vLLM configuration | TP=1, Qwen2.5-0.5B-Instruct, MP/HTTP host ports 15555/39080, chunk size 256, shared 8 GiB L1, three GPU/CPU worker slots, and managed Redis L2. Two engines on `10.0.121.10` shared one server and a third engine on `10.0.75.171` used a second server. Status converged to `desiredServers=readyServers=2` and matched/ready/covered engines `3/3/3`. | +| vLLM store/reset/retrieve | Engine A's 750-token request caused its local server to log `Stored 512`. After `/reset_prefix_cache`, fresh engine B's first matching request caused the same `.121` server to log `Retrieved 512`; that server registered two distinct GPU IDs and affinity keys. Engine C's first matching request caused only its `.75` server to retrieve 512 through Redis L2 before local GPU transfer. | +| Effective SGLang configuration | TP=1, TinyLlama-1.1B-Chat-v1.0, MP/HTTP host ports 15556/39081, chunk size 256, shared 8 GiB L1, two GPU/CPU worker slots, and managed Redis L2. Two engines on `.121` shared one server and a third engine on `.75` used a second server. Status converged to servers `2/2` and engines `3/3/3` ready/covered. | +| SGLang store/flush/retrieve | Engine A's 831-token request caused `Stored 768`; after `/flush_cache`, its response reported `cached_tokens=768` and `host=768` while the server logged `Retrieved 768`. Fresh engine B's first request retrieved the same 768-token entry through the shared `.121` server. Engine C's first matching request retrieved 768 through only its local `.75` server. | +| Same-node and no cross-node CUDA IPC | Every engine endpoint came from its own Downward API `status.hostIP`; each direct server used exact node affinity for that engine node, and no MP Service existed. The `.121` servers registered/transferred only for `.121` engines, while the `.75` servers registered/transferred only for `.75` engines. Cross-node reuse occurred through Redis L2, then terminated in the engine's same-node server; no CUDA IPC endpoint crossed nodes. | +| Host-port conflict | A second backend demanded the same 15555/39080 host ports on `.121`. Its exact-node server remained Pending with scheduler `didn't have free ports`; status reported `NodeLocalHostPortConflict`, desired one server, zero ready servers, and the selected engine uncovered/gated. It did not accept the first backend's listener. | +| Redis outage/recovery | SGLang and vLLM were fault-tested separately with two engine-demanded node-local servers. Deleting each managed Redis Pod changed `RemoteStorageReady` to `False/RemoteStorageUnavailable` while `ConnectorReady` and both local servers remained healthy. The replacement Redis restored the condition to True; all engine/server UIDs and restart counts remained unchanged/zero, and post-recovery reset/flush requests retrieved 512 vLLM or 768 SGLang tokens from shared L1. | +| Engine lifecycle | The initial GPU run proved node-scoped deletion/recreation and gating. Architecture review then selected warm idle retention instead of immediate final-engine deletion. The final controller unit/envtest contract marks the server idle, reuses the same Pod when demand returns within the typed window, deletes it after expiry, and supports explicit zero-retention cleanup. The focused metrics GPU run left the server unchanged across normal engine traffic; it did not repeat the full engine-deletion matrix. | +| Canonical selector follow-up | The latest amd64 controller `sha256:9d15ece6a854655d77c558b30005345190fd9cd5966fc4bedb1385b72cb95a70` rejected a non-canonical `app:` selector and separately rejected a second CacheBackend claiming the existing namespace-local `inferencecache.io/cache-domain`. A canonical vLLM engine matched successfully. The backend had zero server Pods before engine scheduling, then created one exact-node server on `10.0.121.10`; status converged to desired/ready servers `1/1` and matched/ready/covered engines `1/1/1`. | +| Representative common-MP metrics | A focused vLLM TP=1 follow-up used the same pinned vLLM and standalone-server images and checksummed LMCache 0.5.3 wheel. Before traffic, `/metrics` reported L1 usage `0`. A 1,521-token request logged `Stored 1280 tokens`, raised `lmcache_mp_l1_write_chunks_total` to `5`, and raised L1 usage to `15,728,640` bytes. After successful `/reset_prefix_cache`, the identical request logged `Retrieved 1280 tokens in 0.002 seconds`; requested/hit counters became `2560/1280`, and vLLM reported a 42.1% external prefix-cache hit rate. vLLM and SGLang data-plane correctness had already passed separately above; the FastAPI `/metrics` endpoint and counters belong to their common standalone MP server, so this focused follow-up did not repeat the full runtime/node/Redis matrix. | +| Post-rebase engine-metrics merge | A focused current-controller test used controller `sha256:a318ea5e96bbcd0ea10f33394beb8fdaa74ce8a93424f6a38f8df75edb4e6889` and subscriber `sha256:ed8a2ad680d248be3e737adf4ba09cd289f6b99909cec580d1cf240d877b3d88`. Live vLLM admission rendered `--hash-scheme=vllm` and the default `http://127.0.0.1:8000/metrics`. A real SGLang 0.5.13.post1 TP=1 Pod instead used its explicitly configured port 8000 and rendered `--hash-scheme=sglang`, `--engine-metrics-url=http://127.0.0.1:8000/metrics`, and `--enable-metrics`. Its endpoint exposed the expected `sglang:token_usage`, `sglang:cache_hit_rate`, `sglang:num_running_reqs`, and `sglang:num_queue_reqs` families; an active request produced `num_running_reqs=1`. Before engine startup the subscriber reported connection failures and `load_signal_stale`; after the endpoint came up it logged `load_signal_recovered`. The authenticated server snapshot then reported `statsReported=true`, `pressure=0.00390625`, one prefix, and a current update timestamp for the SGLang replica. This validates the rebased per-engine profile selection and custom-port plumbing without repeating the already-completed runtime/node/Redis matrix. | +| UID-scoped two-pool GPU isolation | A final vLLM TP=1 run placed two CacheBackends and two engines on A100 node `10.0.75.171`, using disjoint ports `15655/39180` and `15656/39181`, 8 GiB L1, chunk size 256, and one GPU/CPU worker per pool. The tested controller was `sha256:0320ba07bae7bf5158ca1120e96c8e31275bf0b2e879a89cde46a48d0f8edc9b`; the server was the pinned standalone digest above; the vLLM digest was `sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a`; and the checksummed wheel carrier was `sha256:81b6767d1435f41832d3494eee47f93d08998cba99f50e9b019d6a7ba7ea1e33`. Both gate checks verified the exact full-UID name in declared and effective `/config`. A stored 1,536 tokens; B's first request for the identical prompt still missed and independently stored 1,536, proving it did not retrieve A's object. After each engine's `/reset_prefix_cache`, each server independently retrieved 1,536 tokens. Metrics for each pool were write/read chunks `6/6`, lookup requested/hit tokens `3072/1536`, L1 usage `18,874,368` bytes, and zero L2 adapters. The servers registered different GPUs (`GPU-4d9375ae-f17c-3721-2417-3af8a961c530` and `GPU-eba663df-3529-f1ea-0da8-6fbe522719d9`) and neither registered the other's worker. Recreating B changed its engine/worker identity but preserved B's server Pod UID and L1; its first request retrieved 1,536 from retained B L1 while A remained unchanged. LMCache did not retain a visible UID-named file in `/dev/shm` during this GPU-worker run, so named-inode lifetime is established by the focused CPU tests above; the GPU test establishes effective-config, endpoint, worker-registration, and behavioral data isolation. | +| Cleanup/control-plane restore | All test objects were removed. The original validation restored from `/private/tmp/inference-cache-phase8-engine-first-sjc-backup-20260812`; the focused metrics run restored from `/private/tmp/inference-cache-phase8-metrics-backup-20260812`; the UID-scoped run restored from `/private/tmp/inference-cache-phase8-shm-backup-20260812`; and the post-rebase metrics run restored from `/private/tmp/inference-cache-phase8-rebase-metrics-backup-20260812`. Semantic comparisons of the CRD spec, ClusterRole rules, ClusterRoleBinding role/subjects, controller and server Deployment specs, and both webhook lists returned no differences after the final run. The original controller digest `sha256:6dcab2344027ef8ac3db2ab22352cdaa77d80202ec11df49dddeeefe08095b18` returned `1/1` Ready, and the test namespace had no remaining workload or CacheBackend. | + +- [x] Zero servers before an engine is scheduled; one healthy same-node server + after the first vLLM or SGLang TP=1 engine is placed. +- [x] Multiple same-node engines share one server for both runtimes. +- [x] Engines distributed across two nodes use only their respective same-node + servers; cross-node reuse terminates through Redis L2 without cross-node + CUDA IPC. +- [x] vLLM store → local reset → retrieve and SGLang store → flush → retrieve. +- [x] Desired/ready server counts and per-engine same-node coverage converge. +- [x] Host-port conflict leaves the second server Pending and its engine gated. +- [x] Redis outage/recovery with multiple node-local servers does not restart + engines or healthy local servers. +- [x] Engine lifecycle is node-scoped; typed idle retention reuses the server + before expiry and removes it only after expiry (or immediately at zero). +- [x] Before/after MP `/metrics` snapshots on the representative common server + data path, in addition to the separate vLLM and SGLang functional runs. +- [x] Two CacheBackends with disjoint ports retain distinct UID-scoped SHM + names and inodes on one live node through server startup/replacement and + idle-retention reuse. +- [x] Two GPU engines independently pass store → engine GPU/local KV clear → L1 + retrieve and worker restart/reconnect; exact effective SHM identities, + disjoint worker registrations, and an identical-prompt cold miss prove + that neither pool retrieved or registered the other pool's object/worker. ### Exit criteria -- [ ] Every selected engine Pod is covered by exactly one healthy same-node MP - server. -- [ ] No load-balanced Service can route an engine to another node's server. -- [ ] Shared L1 accounting and failure blast radius are measured. -- [ ] Cross-`CacheBackend` sharing remains rejected. +- [x] Repository, envtest, samples, coverage, and fresh-install validation pass. +- [x] Every selected engine Pod is covered by exactly one healthy same-node MP + server under the engine-first lifecycle. +- [x] No load-balanced Service can route an engine to another node's server. +- [x] Shared L1 accounting and Redis failure blast radius are revalidated with + engine-demanded servers. +- [x] Cross-`CacheBackend` sharing remains rejected by the name+UID demand filter. +- [x] Required vLLM and SGLang functional matrix evidence is supplemented by + before/after metrics from their common standalone MP-server data path. +- [x] UID-scoped NodeLocal SHM isolation passes focused live-node and GPU + validation; missing or mismatched SHM identity never counts as Ready or + covered and never admits an engine. -## Post-migration improvements and additional features +## Required GPU validation matrix -These items are separate capability profiles. They are not Phase 3 or Phase 4 -exit criteria and do not block migration away from the legacy IP data plane: +The matrix grows by phase. A cell is complete only when it proves a cache hit +after clearing or replacing the engine GPU cache; successful process startup is +not sufficient. -- [ ] Design and validate multi-node TP and vLLM distributed-executor profiles, - including connector/server cardinality, endpoint discovery, failure - domains, scheduling, and an explicit admission contract. -- [ ] Design and validate MLA and other model-specific connector profiles using - model architecture metadata rather than image or model-name heuristics. -- [ ] Add client/server compatibility signaling or health detection before - supporting multiple LMCache version baselines; do not generalize from an - arbitrary mismatched-version test pair. -- [ ] Add each profile to the supported validation matrix only after its own - GPU correctness, failure-recovery, and operability gates pass. +| Runtime | Topology | Remote L3 | Parallelism | Required by | +|---|---|---|---|---| +| SGLang | PodLocal | none | TP=1 | Phase 3 | +| vLLM | PodLocal | none | TP=1 | Phase 4 | +| vLLM | PodLocal | none | TP=2 | Phase 4 | +| SGLang | NodeLocal | Redis | multiple engine Pods | Phase 8 | +| vLLM | NodeLocal | Redis | multiple engine Pods | Phase 8 | -### Typed MP Mooncake L2 adapter +Every required data test records: -Mooncake remains a supported provider direction, but its removed implementation -was coupled to the legacy IP connector and is not safe to restore. Future work -must add a new typed MP binding using LMCache's `mooncake_store` L2 adapter: +- exact image digests and LMCache version; +- Kubernetes, driver, CUDA, and GPU model; +- engine args and effective MP server config; +- first-request store evidence; +- GPU-cache clear or fresh-engine proof; +- second-request retrieve/hit evidence; +- MP metrics before and after, plus L3 metrics only when an optional L3 binding + is part of that particular test. -- [ ] Add a provider-specific typed configuration for Mooncake metadata/master - addresses, protocol, segment sizing, local buffer sizing, credentials, - networking, and managed-versus-external lifecycle. -- [ ] Render `--l2-adapter` configuration through the common MP server without - exposing `lm://` or `LMCacheConnectorV1`. -- [ ] Define provider-scoped host-network/RDMA placement and security; do not - reuse an engine-global hostNetwork toggle. -- [ ] Validate cross-Pod sharing, restart/re-registration, failure isolation, - and both vLLM and SGLang client paths against pinned released artifacts. -- [ ] Never translate a legacy Mooncake object to Redis or infer typed adapter - settings from its old URL; migration requires an explicit operator choice. +## Overall definition of done -### Managed backend clusters +The migration is complete only when all of the following are true: -The current managed Redis renderer intentionally creates one standalone Redis -Pod. Multiple replicas behind its Service would be independent keyspaces, not a -cluster. A future managed backend-cluster capability must therefore be -provider-specific: +- [x] `spec.type: LMCache` selects only MP implementations. +- [x] Both SGLang and vLLM pass the required PodLocal GPU matrix. +- [x] Host-only MP is supported for both engines; optional L3 implementations + are validated and versioned independently from the engine connector gate. +- [x] Current MP server health is observable and steady-state cache behavior is + tested. +- [x] `remoteStorage` is optional L3 and no longer contains LMCacheServer. +- [x] No production code injects `LMCacheConnectorV1`, `lm://`, or + `LMCACHE_REMOTE_URL`. +- [x] Remote-L3 lifecycle events do not automatically roll MP engines. +- [x] Every old IP object has been migrated or intentionally deleted. +- [x] Canonical samples, reference manifests, CLI output, and design documents + describe only the implemented MP behavior. +- [x] NodeLocal, if enabled, guarantees same-node server selection and accurate + engine coverage; otherwise it remains rejected rather than partially + accepted. -- [ ] Define Redis topology explicitly (for example standalone versus cluster), - including shard count, replicas per shard, stable identity, discovery, - failover, resharding, persistence, and readiness semantics. -- [ ] Decide whether inference-cache owns those resources directly or composes - with a dedicated Redis operator; keep the core runtime/provider boundary - inference-system-neutral. -- [ ] Verify that the selected LMCache RESP adapter or proxy endpoint supports - the advertised cluster behavior before exposing it in the support matrix. -- [ ] Keep generic `remoteStorage.workload` limited to Pod scheduling/security; - do not add replicas or autoscaling that silently changes provider semantics. +Phase 8 is complete, including focused UID-scoped SHM validation, and is the +final phase of this migration. The future capability profiles below are +independent backlog items rather than additional migration phases. -### Directional LMCache roles for PD separation +## Known limitations and future work -`ReadOnly` / `WriteOnly` remain generic CacheBackend API concepts, but all -LMCache backends currently admit only `ReadWrite`. This is an intentional safety -restriction, not a claim that producer/consumer roles are unnecessary. +These are independent known limitations and future capability profiles, not +additional migration phases. They do not restore the legacy IP data plane and +must not introduce `LMCacheConnectorV1`, `lm://`, or an LMCacheServer provider. +A profile enters the supported matrix only after its API contract, GPU +correctness, failure-recovery, security, and operability gates pass against +immutable artifacts. The numbered items below are the future-work backlog. -| Finding | Evidence/impact | -|---|---| -| SGLang's LMCache integration has no directional role surface. | `--enable-lmcache` always participates in both store and retrieve. | -| vLLM accepts `kv_consumer`, `kv_producer`, and `kv_both`. | These are connector configuration values, not LMCache server roles. | -| LMCache 0.5.3's vLLM MP connector did not enforce the configured direction in live GPU tests. | `kv_consumer` still stored and `kv_producer` still retrieved, so exposing ReadOnly/WriteOnly would create a false API guarantee. | +### 1. NodeLocal hostile-process isolation and aggregate SHM capacity -Future work must treat directional access as a separately validated connector -capability: +Phase 8 owns the accidental-collision fix and its correctness validation: every +NodeLocal pool now uses a deterministic full-UID `--shm-name`, and startup/status +verify that exact identity. This future item covers the stronger security and +capacity guarantees that unique names cannot provide. -- [ ] Define PD producer, consumer, and optional decode write-back semantics, - including whether generated-token KV may be persisted after a request. -- [ ] Adopt a pinned connector that prevents store in consumer mode and retrieve - in producer mode rather than relying only on configuration naming. -- [ ] Add GPU negative tests that fail on any prohibited request, plus normal - prefill-to-decode transfer and multi-turn write-back tests where selected. -- [ ] Lift LMCache admission restrictions only for an adapter/version profile - that passes those tests; do not infer support from engine CLI acceptance. +- [ ] Define the hostile-process boundary. A unique name does not stop a + same-node process with host `/dev/shm` access and compatible Unix + credentials from deliberately opening or unlinking another pool. Decide + whether production support requires distinct Unix identities, isolated + SHM backing, admission-enforced node separation, or a combination. +- [ ] Account for aggregate host `/dev/shm` capacity across co-located pools and + expose actionable admission/Pending/status behavior before publishing a + supported multi-pool capacity envelope. +- [ ] Validate the selected tenant boundary with unauthorized open/unlink tests; + until then, multiple pools on one node are supported only inside one + mutually trusted node domain. + +### 2. MP client/server compatibility signaling + +- [ ] Add client/server compatibility signaling or health detection before + supporting multiple LMCache version baselines. +- [ ] Define the admitted version/digest relationship and surface actionable + mismatch status rather than inferring compatibility from process health. +- [ ] Do not generalize support from an arbitrary mismatched-version test pair; + qualify each selected combination with GPU store/clear/retrieve and + recovery evidence. -### LMCache connector control-plane convergence and SGLang TP>1 +### 3. SGLang TP>1 control-plane convergence SGLang TP>1 is outside the migration baseline. Inference-cache neither patches the engine-owned connector nor adds a TP=1 admission guard. @@ -1014,8 +1180,6 @@ the engine-owned connector nor adds a TP=1 admission guard. | SGLang 0.5.3 runs the control flow in every TP rank. | In TP=2, one rank consumed the exactly-once prefetch result; the other got `Prefetch job ... not found`, so the cross-rank minimum became zero. | | A diagnostic rank-0-owner overlay retrieved 1,280 tokens on both ranks. | It proves a coordination direction, not a safe production patch; collective failure/cancellation remains undesigned. | -Future work must answer the architectural question before selecting a fix: - - [ ] Determine why the vLLM and SGLang integrations deliberately use different scheduler/worker ownership models and whether SGLang exposes a stable scheduler-to-worker metadata path suitable for LMCache. @@ -1027,9 +1191,98 @@ Future work must answer the architectural question before selecting a fix: Redis-backed hit, partial hit, timeout, cancellation, and lock/session cleanup. - [ ] Adopt only an immutable released connector artifact, then add SGLang - TP>1 back to the production validation matrix after the tests pass. + TP>1 to the production validation matrix after the tests pass. + +### 4. Multi-node TP and distributed executors -### LMCache MP server restart and connector re-registration +- [ ] Define connector and MP-server cardinality for vLLM distributed executors + and multi-node TP without allowing cross-node CUDA IPC. +- [ ] Specify deterministic endpoint discovery, engine/server placement, + scheduling ownership, failure domains, and admission behavior. +- [ ] Validate node loss, rank loss, partial registration, replacement, and + store/clear/retrieve across all supported ranks before adding a profile to + the matrix. + +### 5. MLA and architecture-specific connectors + +- [ ] Define an explicit typed capability contract for MLA and other + model-specific connector behavior. +- [ ] Use authoritative model architecture metadata; never infer support from + image names, model-name strings, or runtime labels. +- [ ] Add architecture-specific correctness and incompatibility tests before + admission accepts the profile. + +### 6. Directional LMCache roles for PD separation + +`ReadOnly` / `WriteOnly` remain generic CacheBackend API concepts, but all +LMCache backends currently admit only `ReadWrite`. This is an intentional safety +restriction, not a claim that producer/consumer roles are unnecessary. + +| Finding | Evidence/impact | +|---|---| +| SGLang's LMCache integration has no directional role surface. | `--enable-lmcache` always participates in both store and retrieve. | +| vLLM accepts `kv_consumer`, `kv_producer`, and `kv_both`. | These are connector configuration values, not LMCache server roles. | +| LMCache 0.5.3's vLLM MP connector did not enforce the configured direction in live GPU tests. | `kv_consumer` still stored and `kv_producer` still retrieved, so exposing ReadOnly/WriteOnly would create a false API guarantee. | + +- [ ] Define PD producer, consumer, and optional decode write-back semantics, + including whether generated-token KV may be persisted after a request. +- [ ] Adopt a pinned connector that prevents store in consumer mode and retrieve + in producer mode rather than relying only on configuration naming. +- [ ] Add GPU negative tests that fail on any prohibited request, plus normal + prefill-to-decode transfer and multi-turn write-back tests where selected. +- [ ] Lift LMCache admission restrictions only for an adapter/version profile + that passes those tests; do not infer support from engine CLI acceptance. + +### 7. Typed MP Mooncake L2 adapter + +Mooncake remains a supported provider direction, but its removed implementation +was coupled to the legacy IP connector and is not safe to restore. Future work +must add a new typed MP binding using LMCache's `mooncake_store` L2 adapter: + +- [ ] Add a provider-specific typed configuration for Mooncake metadata/master + addresses, protocol, segment sizing, local buffer sizing, credentials, + networking, and managed-versus-external lifecycle. +- [ ] Render `--l2-adapter` configuration through the common MP server without + exposing `lm://` or `LMCacheConnectorV1`. +- [ ] Define provider-scoped host-network/RDMA placement and security; do not + reuse an engine-global hostNetwork toggle. +- [ ] Validate cross-Pod sharing, restart/re-registration, failure isolation, + and both vLLM and SGLang client paths against pinned released artifacts. +- [ ] Never translate a legacy Mooncake object to Redis or infer typed adapter + settings from its old URL; migration requires an explicit operator choice. + +### 8. Managed backend topology and security + +The current managed Redis renderer intentionally creates one standalone Redis +Pod. Multiple replicas behind its Service would be independent keyspaces, not a +cluster. A future managed backend-cluster capability must therefore be +provider-specific. Its current security boundary is also limited: password +authentication is optional, no backend NetworkPolicy is controller-owned, and +the pinned LMCache 0.5.3 RESP adapter does not support TLS. The existing managed +Redis profile is therefore suitable only for an explicitly trusted development +or private network, not as a secure multi-tenant production profile. + +- [ ] Define Redis topology explicitly (for example standalone versus cluster), + including shard count, replicas per shard, stable identity, discovery, + failover, resharding, persistence, and readiness semantics. +- [ ] Decide whether inference-cache owns those resources directly or composes + with a dedicated Redis operator; keep the core runtime/provider boundary + inference-system-neutral. +- [ ] Verify that the selected LMCache RESP adapter or proxy endpoint supports + the advertised cluster behavior before exposing it in the support matrix. +- [ ] Keep generic `remoteStorage.workload` limited to Pod scheduling/security; + do not add replicas or autoscaling that silently changes provider semantics. +- [ ] Define whether authentication is mandatory for a production profile and + keep all credentials in namespace-local Secret references. +- [ ] Define NetworkPolicy ownership and ingress/egress selectors for managed + Redis; do not assume that a ClusterIP is a tenancy boundary. +- [ ] Require a pinned TLS-capable LMCache RESP client, a verified TLS proxy, or + an equivalently explicit encrypted transport before advertising Redis + across an untrusted network. Continue rejecting inert TLS configuration. +- [ ] Validate unauthorized access denial, credential rotation, network-policy + isolation, and selected encrypted-transport failure/recovery behavior. + +### 9. LMCache MP server restart and connector re-registration The migration guarantees steady-state MP operation and sidecar process-health observation. It does not guarantee that a running engine continues caching after @@ -1057,122 +1310,65 @@ If this capability is selected later, its independent scope is: - [ ] Validate crash, hang, fast restart, repeated restart, callback failure, TP=1/TP=2, and post-recovery store/flush/retrieve for each selected engine profile. -- [ ] For NodeLocal, validate DaemonSet rollout and single-node server restart - separately from the basic same-node topology. - -## Required GPU validation matrix - -The matrix grows by phase. A cell is complete only when it proves a cache hit -after clearing or replacing the engine GPU cache; successful process startup is -not sufficient. - -| Runtime | Topology | Remote L3 | Parallelism | Required by | -|---|---|---|---|---| -| SGLang | PodLocal | none | TP=1 | Phase 3 | -| vLLM | PodLocal | none | TP=1 | Phase 4 | -| vLLM | PodLocal | none | TP=2 | Phase 4 | -| SGLang | NodeLocal | Redis | multiple engine Pods | Phase 8 | -| vLLM | NodeLocal | Redis | multiple engine Pods | Phase 8 | - -Every required data test records: - -- exact image digests and LMCache version; -- Kubernetes, driver, CUDA, and GPU model; -- engine args and effective MP server config; -- first-request store evidence; -- GPU-cache clear or fresh-engine proof; -- second-request retrieve/hit evidence; -- MP metrics before and after, plus L3 metrics only when an optional L3 binding - is part of that particular test. - -## Test pyramid - -| Layer | Required evidence | -|---|---| -| API/unit | schema, defaulting, validation, provider matrix, deep copy, status transitions | -| Renderer/unit | exact args/env/config, resources, probes, security, volumes, idempotence, collision rejection | -| Envtest | real CREATE/UPDATE admission and status persistence; legacy grandfathering only if Phase 6 activates | -| Kubernetes smoke | live webhook injection into matching engine Pods, native-sidecar schema support, controller-owned workload shape | -| GPU functional | store/flush/retrieve and cross-Pod L3 reuse | -| GPU fault | Redis loss/recovery, engine rollout, node drain for NodeLocal | -| Upgrade/migration | Repository manifest conversion by default; old-object inventory, dry-run conversion, grandfather rules, and rollback only if Phase 6 activates | - -## Security, reliability, scalability, and cost gates - -### Security - -- [ ] No production managed Redis profile is exposed without an explicit network - isolation and credential/TLS posture. -- [x] Secrets are referenced, not embedded in CR status, Pod args visible to all - readers, logs, or Events. -- [ ] PodLocal and NodeLocal GPU visibility is documented and reviewed for the - target tenancy model. -- [ ] NodeLocal host networking/IPC is an explicit operator choice. -- [ ] Cross-namespace remote endpoints retain explicit opt-in validation. - -### Reliability - -- [x] MP server health affects connector status. -- [ ] Remote L3 loss follows tested fail-open/fail-closed behavior. - -### Scalability and latency - -- [x] PodLocal memory cost is reported per engine Pod. -- [ ] NodeLocal memory cost is reported per node. -- [ ] Worker pool sizing is tested under the expected engine count and TP shape. -- [ ] Remote L3 concurrency and connection limits are bounded. -- [x] Routing/index signals can be correlated with actual LMCache hit metrics. - -### Operability - -- [x] Status distinguishes connector, MP server, engine, and remote L3 health. -- [ ] Metrics expose server availability, L1/L3 store/retrieve/hit, capacity, - and eviction. -- [ ] Events contain an actionable remediation or migration instruction. -- [ ] Samples never depend on an implicit runtime-selected connector mode. - -## Risk register - -| Risk | Impact | Mitigation / gate | -|---|---|---| -| A legacy consumer appears after Phase 0 | Breaking removal | Reconfirm before removal; activate Phase 6 and a grandfather period when non-zero. | -| MP client/server version skew | Permanent unhealthy or protocol failure | Pin a validated client/server baseline and record exact artifacts; automatic version negotiation/detection is a future improvement. | -| Redis restart rolls all engines | Availability blast radius | Lifecycle-specific restart policy in Phase 2. | -| `failOpen` is only a custom env | Contract not enforced | Render native runtime policy and fault-test it. | -| Sidecar sees all node GPUs | Isolation exposure | Document/review tenant model; prefer dedicated nodes where required. | -| PodLocal duplicates CPU L2 | Memory cost per replica | Explicit per-Pod capacity; NodeLocal follow-up. | -| NodeLocal port collision | DaemonSet Pods fail or bind incorrectly | Declared host port, typed port, controller condition, negative tests. | -| NodeLocal engine reaches remote node | CUDA IPC failure | Node-derived endpoint; reject load-balanced service topology. | -| Existing engine-side Mooncake config is treated as MP-equivalent | Admission succeeds but adapter cannot start | No automatic migration; separate MP + Mooncake Store implementation. | -| Index says warm while MP/L3 evicted data | Routing quality degrades silently | Correlate cache events with LMCache metrics/health; define stale-entry behavior. | - -## Roadmap maintenance - -Each implementation PR updates the delivery table and the affected phase's -checkboxes. A phase becomes complete only when every exit criterion is checked; -validation details stay summarized in the phase evidence table rather than in -dated closure sections or separate phase documents. - -## Overall definition of done - -The migration is complete only when all of the following are true: - -- [x] `spec.type: LMCache` selects only MP implementations. -- [x] Both SGLang and vLLM pass the required PodLocal GPU matrix. -- [x] Host-only MP is supported for both engines; optional L3 implementations - are validated and versioned independently from the engine connector gate. -- [x] Current MP server health is observable and steady-state cache behavior is - tested. -- [x] `remoteStorage` is optional L3 and no longer contains LMCacheServer. -- [x] No production code injects `LMCacheConnectorV1`, `lm://`, or - `LMCACHE_REMOTE_URL`. -- [x] Remote-L3 lifecycle events do not automatically roll MP engines. -- [x] Every old IP object has been migrated or intentionally deleted. -- [x] Canonical samples, reference manifests, CLI output, and design documents - describe only the implemented MP behavior. -- [x] NodeLocal, if enabled, guarantees same-node server selection and accurate - engine coverage; otherwise it remains rejected rather than partially - accepted. +- [ ] For NodeLocal, validate per-node server-Pod replacement and single-node + server restart separately from the basic same-node topology. + +### 10. LMCache fail-open and fail-closed semantics + +`spec.integration.failOpen` currently drives status, Events, and the injected +`INFERENCECACHE_FAIL_OPEN` environment variable. The pinned vLLM and SGLang +LMCache MP connectors have not been shown to consume that project-specific +variable or enforce request behavior. The SJC Redis outage tests validated the +default fail-open status path while L1 remained available; they did not validate +an L1+L2 outage or a fail-closed request failure. Consequently, accepting +`failOpen: false` is not yet proof of a data-plane fail-closed guarantee. + +- [ ] Decide whether LMCache admission must temporarily accept only + `failOpen: true` until a connector-native fail-closed surface exists. +- [ ] Map the API to a pinned connector feature that actually controls request + behavior; do not treat a project-specific environment mirror as + enforcement. +- [ ] Validate vLLM and SGLang with L1 available/L2 unavailable, L1 + unavailable/L2 available, and complete L1+L2 loss. +- [ ] Prove that fail-open recomputes locally without failing the request and + that fail-closed fails within a bounded timeout with accurate status and + Events. +- [ ] Test recovery without engine restart and remove the restriction only for + runtime/version profiles that pass the same fault matrix. + +### 11. MP worker-pool saturation and sizing + +Phase 8 proved that multiple TP=1 engine Pods can share one NodeLocal server and +requires `maxGPUWorkers` to cover their count. It did not establish saturation, +queueing, backpressure, or latency behavior at and beyond the configured worker +limits, nor how a future TP profile contributes workers. + +- [ ] Define whether each engine instance, process, or TP rank consumes a GPU + worker and document the corresponding `maxGPUWorkers` formula. +- [ ] Define the purpose and sizing rule for `maxCPUWorkers`, including its + interaction with GPU workers and L2 operations. +- [ ] Load-test below, at, and above both worker limits and record queueing, + rejection, timeout, throughput, and tail-latency behavior. +- [ ] Surface an actionable signal when registered demand reaches or exceeds a + worker limit; silent request hangs are not an acceptable capacity policy. +- [ ] Qualify each future TP/distributed profile independently rather than + extrapolating from TP=1 engine counts. + +### 12. Remote L3 concurrency and connection limits + +The current Redis/RESP profile has no typed or validated bound for connections +and concurrent store/retrieve work as the number of engine Pods, NodeLocal +servers, and nodes grows. This is independent of whether Redis remains a +singleton or later becomes a managed cluster. + +- [ ] Measure and document Redis connections per MP server and per registered + engine under idle, store, retrieve, and reconnect behavior. +- [ ] Define client-side connection, concurrency, queue, and timeout limits and + their relationship to Redis `maxclients` and server resources. +- [ ] Validate connection exhaustion, slow Redis, concurrent store/retrieve, + reconnect storms, and recovery without engine rollout. +- [ ] Add capacity guidance and actionable status/metrics for connection or + concurrency exhaustion before advertising a production scale envelope. ## Upstream references diff --git a/docs/design/lmcache-server-persistence.md b/docs/design/lmcache-server-persistence.md index 7b49bece..da62c4f0 100644 --- a/docs/design/lmcache-server-persistence.md +++ b/docs/design/lmcache-server-persistence.md @@ -40,11 +40,12 @@ PVC**: volume mounted on the server pod. Provisioning a PVC for that server would mount storage nothing writes KV to. 2. **LMCache's only on-server local-disk path is node-local.** Its MP-mode (the - L2 NIXL POSIX backend writing to a `file_path`) is documented to deploy as a - DaemonSet with `hostNetwork` and a shared host `/dev/shm`, where the control - socket is ZMQ-only and KV bytes move over CUDA-IPC or POSIX shared memory. A - server reachable only through a ClusterIP Service therefore has **no data - plane** in that mode, and the mode is **per-node, not per-`CacheBackend`**. + L2 NIXL POSIX backend writing to a `file_path`) requires `hostNetwork` and a + shared host `/dev/shm`, where the control socket is ZMQ-only and KV bytes move + over CUDA-IPC or POSIX shared memory. A server reachable only through a + ClusterIP Service therefore has **no data plane** in that mode. The current + implementation creates one directly scheduled server Pod per active engine + node and CacheBackend; multiple pools on one node require disjoint host ports. MP-mode is thus incompatible with this project's per-backend Deployment + ClusterIP, engines-anywhere model. diff --git a/docs/design/sglang-lmcache-mp-mode.md b/docs/design/sglang-lmcache-mp-mode.md index ac9fb66b..b59c1e82 100644 --- a/docs/design/sglang-lmcache-mp-mode.md +++ b/docs/design/sglang-lmcache-mp-mode.md @@ -10,15 +10,16 @@ MP migration. The current contract is in > MP viability. Its flat worker fields, engine-image worker default, vLLM IP > coexistence, and predictions that vLLM MP was future work are historical and > were physically removed in migration Phase 7. Current production behavior is -> the typed PodLocal API and common MP-server sidecar renderer defined by +> the typed PodLocal/NodeLocal API and common MP renderer defined by > [`lmcache-multiprocess-migration-roadmap.md`](lmcache-multiprocess-migration-roadmap.md) > and [`cachebackend-api.md`](cachebackend-api.md). ## Current-state summary -- vLLM and SGLang support only typed PodLocal LMCache MP in the current API. -- Admission injects a CacheBackend-configured `lmcache-mp-server` native sidecar; - it does not own or replace the engine image. +- vLLM and SGLang support typed PodLocal and NodeLocal LMCache MP in the current + API. PodLocal injects a native sidecar; NodeLocal follows scheduled engines + with one on-demand same-node server Pod. +- Admission does not own or replace the engine image. - Omitting `remoteStorage` selects host-only MP. Explicit Redis selects the only currently supported remote L3. Removed LMCacheServer and legacy IP-wired Mooncake objects are never translated to Redis; a new typed MP Mooncake L2 diff --git a/docs/quickstart.md b/docs/quickstart.md index e6bd1e6b..38549ec4 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -20,7 +20,7 @@ spec: type: LMCache # engine-local cache integration engineSelector: matchLabels: - app: my-engine # must match your engine pods' labels + inferencecache.io/cache-domain: my-engine-cache lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -41,16 +41,34 @@ spec: modelID: Qwen/Qwen2.5-0.5B-Instruct ``` -That is the whole CacheBackend. The runtime/cache pair, matching labels, model +That is the whole CacheBackend. The runtime/cache pair, ownership domain, model identity, and MP server contract are explicit; `integration.role` defaults to `ReadWrite`, the readiness gate's `firstEventTimeout` defaults to `5m`, and `integration.failOpen` defaults to `true`. Omitting `remoteStorage` selects host-only MP: L1 is per engine Pod and there is no cross-Pod sharing. Add an explicit Redis L3 when sharing is required. -> **One label does the binding.** The value under -> `engineSelector.matchLabels` must also appear on your engine pods' -> template labels. That label match is what lets the mutating Pod webhook +To share one MP L1 across several engine Pods on each GPU node, use the +focused NodeLocal samples for +[vLLM](../config/samples/cachebackend-vllm-nodelocal-host-only.yaml) or +[SGLang](../config/samples/cachebackend-sglang-nodelocal-host-only.yaml). +NodeLocal is an advanced host-bound topology. The inference system schedules +each engine without CacheBackend changing its placement; the controller then +creates one shared server Pod on every node that actually has an active +selected engine. The server mounts host `/dev/shm` and declares MP and HTTP host +ports. Engines are held in an init gate until their own node's server reports +the exact CacheBackend name/UID/generation and healthy config. Co-schedule only +mutually trusted engines and enforce host-port access with node firewalls; +hostNetwork bypasses Kubernetes NetworkPolicy. L1 capacity is per node, and +`maxGPUWorkers` must cover the maximum engine instances on that node. The +focused samples retain an idle per-node server and its L1 for 300 seconds so an +engine restart can reuse them; set `idleRetentionSeconds: 0` for immediate +cleanup. + +> **One label does the binding.** Every non-empty selector contains only +> `inferencecache.io/cache-domain`; its namespace-scoped value must also appear +> on your engine pods' template labels. Other Pod labels do not participate in +> CacheBackend ownership. That domain match lets the mutating Pod webhook > inject the cache wiring at pod CREATE. Drift them apart and the engine runs > uncached — `kubectl get cachebackend` then shows `MATCHED: 0`. @@ -143,8 +161,8 @@ Once the backend is Ready and engine pods are bound, three things are live: annotation. Any active gate that reports a per-stage failure can hold the backend at `Ready=False` with a stage-specific reason on `.status.conditions[]` — see [Troubleshooting](#troubleshooting). - `ENDPOINT` is empty for host-only PodLocal MP and contains only a configured - remote L3 endpoint, never the loopback MP connector address. `MATCHED` is the + `ENDPOINT` is empty for host-only PodLocal or NodeLocal MP and contains only a + configured remote L3 endpoint, never a local MP connector address. `MATCHED` is the engine-pod count the selector binds, and `PREFIXES` / `LASTEVENT` show the cache actually receiving state. diff --git a/docs/reference-stack/manifests/deployment.yaml b/docs/reference-stack/manifests/deployment.yaml index 8cf59ac8..c64e3b5c 100644 --- a/docs/reference-stack/manifests/deployment.yaml +++ b/docs/reference-stack/manifests/deployment.yaml @@ -20,7 +20,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: vllm-lmcache-llama-8b + inferencecache.io/cache-domain: vllm-lmcache-llama-8b lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -45,9 +45,10 @@ kind: Deployment metadata: name: vllm-lmcache-llama-8b namespace: cache-substrate - labels: - app: vllm-lmcache-llama-8b - inferencecache.io/backend-type: lmcache + labels: + app: vllm-lmcache-llama-8b + inferencecache.io/cache-domain: vllm-lmcache-llama-8b + inferencecache.io/backend-type: lmcache inferencecache.io/engine: vllm spec: replicas: 1 @@ -58,6 +59,7 @@ spec: metadata: labels: app: vllm-lmcache-llama-8b + inferencecache.io/cache-domain: vllm-lmcache-llama-8b inferencecache.io/backend-type: lmcache inferencecache.io/engine: vllm spec: diff --git a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml index e68819c5..b7f316fd 100644 --- a/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml +++ b/docs/reference-stack/manifests/sglang-lmcache/deployment.yaml @@ -120,7 +120,7 @@ spec: role: ReadWrite engineSelector: matchLabels: - app: sglang-lmcache-llama-8b + inferencecache.io/cache-domain: sglang-lmcache-llama-8b lmCache: topology: PodLocal chunkSizeTokens: 256 @@ -164,6 +164,7 @@ spec: labels: app: sglang-lmcache-llama-8b app.kubernetes.io/name: sglang + inferencecache.io/cache-domain: sglang-lmcache-llama-8b spec: containers: # Container name MUST be `sglang` so the adapter's InjectEngineConfig / diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index ca91b16c..f2f8ae62 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -166,7 +166,7 @@ done kubectl create namespace "$SMOKE_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -log "creating typed PodLocal MP backends" +log "creating typed PodLocal and NodeLocal MP backends" cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply -f - >/dev/null apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend @@ -177,7 +177,7 @@ spec: type: LMCache engineSelector: matchLabels: - app: mp-engine + inferencecache.io/cache-domain: mp-engine integration: role: ReadWrite lmCache: @@ -205,7 +205,7 @@ spec: type: LMCache engineSelector: matchLabels: - app: sglang-engine + inferencecache.io/cache-domain: sglang-engine integration: role: ReadWrite lmCache: @@ -230,7 +230,71 @@ spec: kubernetes.io/os: linux terminationGracePeriodSeconds: 45 redis: {} +--- +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: node-local +spec: + runtime: VLLM + type: LMCache + engineSelector: + matchLabels: + inferencecache.io/cache-domain: node-local-engine + integration: + role: ReadWrite + lmCache: + topology: NodeLocal + chunkSizeTokens: 256 + nodeLocal: + idleRetentionSeconds: 300 + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5556 + httpPort: 8081 + l1Capacity: 1Gi + maxGPUWorkers: 2 + maxCPUWorkers: 2 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + memory: 2Gi +EOF + +log "checking duplicate same-namespace cache domains are rejected" +if cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -f - >/dev/null 2>&1 +apiVersion: inferencecache.io/v1alpha1 +kind: CacheBackend +metadata: + name: overlapping-selector +spec: + runtime: VLLM + type: LMCache + engineSelector: + matchLabels: + inferencecache.io/cache-domain: mp-engine + integration: + role: ReadWrite + lmCache: + topology: PodLocal + podLocal: + server: + image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 + port: 5555 + l1Capacity: 1Gi + maxWorkers: 1 + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + memory: 2Gi EOF +then + fail "duplicate cache-domain engineSelector was admitted" +fi for _ in $(seq 1 60); do endpoint="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend managed-redis -o jsonpath='{.status.remoteStorage.endpoint}' 2>/dev/null || true)" @@ -245,6 +309,13 @@ managed_os="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o json managed_grace="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}')" [ "$managed_grace" = "45" ] || fail "managed workload terminationGracePeriodSeconds was not rendered" +log "checking NodeLocal creates no speculative server before an engine is scheduled" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pods -l inferencecache.io/lmcache-node-server=true --no-headers 2>/dev/null | wc -l | tr -d ' ')" = "0" ] \ + || fail "NodeLocal created a server without scheduled engine demand" +if kubectl -n "$SMOKE_NAMESPACE" get service node-local >/dev/null 2>&1; then + fail "NodeLocal MP endpoint must not have a Service" +fi + log "checking real Pod admission renders only the MP wire" pod_json="$tmpdir/admitted-pod.json" cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -o json -f - >"$pod_json" @@ -253,8 +324,10 @@ kind: Pod metadata: name: mp-engine labels: - app: mp-engine + inferencecache.io/cache-domain: mp-engine spec: + nodeSelector: + kubernetes.io/os: linux containers: - name: vllm image: busybox:1.36 @@ -270,6 +343,69 @@ for retired in LMCacheConnectorV1 LMCACHE_REMOTE_URL LMCACHE_REMOTE_SERDE 'lm:// fi done +log "checking real Pod admission renders the same-node NodeLocal wire and startup gate" +node_pod_json="$tmpdir/admitted-node-local-pod.json" +cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -o json -f - >"$node_pod_json" +apiVersion: v1 +kind: Pod +metadata: + name: node-local-engine + labels: + inferencecache.io/cache-domain: node-local-engine +spec: + nodeSelector: + kubernetes.io/os: linux + containers: + - name: vllm + image: busybox:1.36 + command: ["sh", "-c", "sleep 3600"] +EOF +grep -Fq 'lmcache-node-local-gate' "$node_pod_json" || fail "NodeLocal ownership/health startup gate was not injected" +grep -Fq 'EXPECTED_SHM_NAME' "$node_pod_json" || fail "NodeLocal startup gate does not verify UID-scoped shared memory" +grep -Fq 'INFERENCECACHE_NODE_IP' "$node_pod_json" || fail "NodeLocal hostIP Downward API was not injected" +grep -Fq 'status.hostIP' "$node_pod_json" || fail "NodeLocal endpoint is not derived from status.hostIP" +grep -Fq 'kubernetes.io/os' "$node_pod_json" || fail "inference-owned nodeSelector was not preserved" +if grep -Fq 'podAffinity' "$node_pod_json"; then + fail "NodeLocal injection unexpectedly added server-first PodAffinity" +fi +grep -Fq 'hostPath' "$node_pod_json" || fail "NodeLocal engine did not receive host /dev/shm" +grep -Fq 'tcp://$(INFERENCECACHE_NODE_IP)' "$node_pod_json" || fail "vLLM NodeLocal connector does not use the node-derived endpoint" +if grep -Fq '"name": "lmcache-mp-server"' "$node_pod_json"; then + fail "NodeLocal engine unexpectedly received the PodLocal native sidecar" +fi + +log "checking scheduled engine demand creates one exact-node server Pod" +kubectl -n "$SMOKE_NAMESPACE" apply -f "$node_pod_json" >/dev/null +for _ in $(seq 1 60); do + engine_node="$(kubectl -n "$SMOKE_NAMESPACE" get pod node-local-engine -o jsonpath='{.spec.nodeName}' 2>/dev/null || true)" + server_name="$(kubectl -n "$SMOKE_NAMESPACE" get pods -l inferencecache.io/lmcache-node-server=true -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + [ -n "$engine_node" ] && [ -n "$server_name" ] && break + sleep 1 +done +[ -n "${engine_node:-}" ] && [ -n "${server_name:-}" ] || fail "scheduled engine did not create an on-demand NodeLocal server" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-target-node}')" = "$engine_node" ] \ + || fail "NodeLocal server target does not match the engine-selected node" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.hostNetwork}')" = "true" ] \ + || fail "NodeLocal server does not use hostNetwork" +node_local_host_ipc="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.hostIPC}')" +[ -z "$node_local_host_ipc" ] || [ "$node_local_host_ipc" = "false" ] \ + || fail "NodeLocal server unexpectedly uses hostIPC" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.volumes[0].hostPath.path}')" = "/dev/shm" ] \ + || fail "NodeLocal server does not mount host /dev/shm" +node_local_backend_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend node-local -o jsonpath='{.metadata.uid}')" +node_local_shm_name="lmcache_l1_pool_inferencecache_${node_local_backend_uid}" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-shm-name}')" = "$node_local_shm_name" ] \ + || fail "NodeLocal server does not carry its UID-scoped shared-memory identity" +node_local_shm_arg="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" \ + -o jsonpath='{range .spec.containers[0].args[*]}{.}{"\n"}{end}' | \ + awk 'previous == "--shm-name" { print; exit } { previous = $0 }')" +[ "$node_local_shm_arg" = "$node_local_shm_name" ] \ + || fail "NodeLocal server does not pass its UID-scoped --shm-name: $node_local_shm_arg" +node_local_ports="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{range .spec.containers[0].ports[*]}{.containerPort}:{.hostPort}{" "}{end}')" +[ "$node_local_ports" = "5556:5556 8081:8081 " ] || fail "NodeLocal host ports were not declared: $node_local_ports" +node_affinity_target="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchFields[0].values[0]}')" +[ "$node_affinity_target" = "$engine_node" ] || fail "server does not use scheduler-bound exact-node affinity" + log "checking current samples against the live CRDs and admission webhooks" sample_namespace="${SMOKE_NAMESPACE}-samples" kubectl create namespace "$sample_namespace" --dry-run=client -o yaml | kubectl apply -f - >/dev/null @@ -294,15 +430,19 @@ for sample in \ config/samples/cache_v1alpha1_cachepolicy.yaml \ config/samples/cache_v1alpha1_cachetenant.yaml \ config/samples/cache_v1alpha1_prompttemplate.yaml \ - config/samples/cache_v1alpha1_pdtopology.yaml \ - config/samples/cachebackend-events-only.yaml \ - config/samples/cachebackend-sglang-hicache.yaml; do + config/samples/cache_v1alpha1_pdtopology.yaml; do kubectl -n "$sample_namespace" apply -f "$sample" >/dev/null done kubectl -n "$sample_namespace" get cachepolicy cachepolicy-sample >/dev/null kubectl -n "$sample_namespace" get cachetenant cachetenant-sample >/dev/null kubectl -n "$sample_namespace" get prompttemplate prompttemplate-sample >/dev/null kubectl -n "$sample_namespace" get pdtopology pdtopology-sample >/dev/null + +for sample in \ + config/samples/cachebackend-events-only.yaml \ + config/samples/cachebackend-sglang-hicache.yaml; do + kubectl -n "$sample_namespace" apply -f "$sample" >/dev/null +done for engine_local in cachebackend-events-only sglang-hicache; do if kubectl -n "$sample_namespace" get deployment "$engine_local" >/dev/null 2>&1 || \ kubectl -n "$sample_namespace" get service "$engine_local" >/dev/null 2>&1; then @@ -324,6 +464,6 @@ log "re-applying the bundle as an idempotent upgrade check" kubectl apply -k "$tmpdir/config/default" >/dev/null kubectl -n "$SYSTEM_NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ deployment/inference-cache-controller-manager deployment/inference-cache-server -kubectl -n "$SMOKE_NAMESPACE" get cachebackend host-only managed-redis >/dev/null +kubectl -n "$SMOKE_NAMESPACE" get cachebackend host-only managed-redis node-local >/dev/null -log "PASS: default install, control-plane APIs, server surfaces, samples, doctor, typed MP admission, managed Redis, and idempotent re-apply" +log "PASS: default install, control-plane APIs, server surfaces, samples, doctor, PodLocal/NodeLocal typed MP admission, engine-demanded NodeLocal server Pods, managed Redis, and idempotent re-apply" diff --git a/docs/reference-stack/scripts/phase5_upgrade_smoke.sh b/docs/reference-stack/scripts/phase5_upgrade_smoke.sh deleted file mode 100755 index 95fd5ffa..00000000 --- a/docs/reference-stack/scripts/phase5_upgrade_smoke.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: 2026 The inference-cache Authors -# -# SPDX-License-Identifier: Apache-2.0 - -# Installs the last Phase 5 revision, creates only Phase 5 typed PodLocal MP -# objects, then upgrades the CRD and controller to the current checkout. This -# intentionally does not create or migrate legacy IP objects: the recorded -# Phase 0/5 consumer audit found no supported population for them. - -set -euo pipefail - -PHASE5_COMMIT="${PHASE5_COMMIT:-10178558bfca308ee3a4b0d584efe4ed3b91197d}" -PHASE5_TAG="${PHASE5_TAG:-phase5-upgrade-base}" -TAG="${TAG:-${GITHUB_SHA:-$(git rev-parse HEAD)}}" -REGISTRY="${REGISTRY:-ghcr.io/cachebox-project}" -KIND_CLUSTER="${KIND_CLUSTER:-ic-phase5-upgrade}" -SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-inference-cache-system}" -SMOKE_NAMESPACE="${SMOKE_NAMESPACE:-ic-phase5-upgrade}" -CERT_MANAGER_VERSION="${CERT_MANAGER_VERSION:-v1.16.1}" -READY_TIMEOUT="${READY_TIMEOUT:-180s}" -KEEP_CLUSTER="${KEEP_CLUSTER:-0}" -LOG_DIR="${LOG_DIR:-/tmp/phase5-upgrade-smoke-logs}" - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" -CURRENT_CONTROLLER_IMG="$REGISTRY/inference-cache-controller:$TAG" -CURRENT_SERVER_IMG="$REGISTRY/inference-cache-server:$TAG" -PHASE5_CONTROLLER_IMG="$REGISTRY/inference-cache-controller:$PHASE5_TAG" -PHASE5_SERVER_IMG="$REGISTRY/inference-cache-server:$PHASE5_TAG" -KIND="${KIND:-$REPO_ROOT/bin/kind}" -[ -x "$KIND" ] || KIND=kind - -log() { printf '[phase5-upgrade-smoke] %s\n' "$*"; } -fail() { printf '[phase5-upgrade-smoke] ERROR: %s\n' "$*" >&2; exit 1; } - -for binary in docker git kubectl tar "$KIND"; do - command -v "$binary" >/dev/null 2>&1 || fail "missing required tool: $binary" -done -git -C "$REPO_ROOT" cat-file -e "$PHASE5_COMMIT^{commit}" \ - || fail "Phase 5 commit is unavailable: $PHASE5_COMMIT" - -mkdir -p "$LOG_DIR" -tmpdir="$(mktemp -d)" -created_cluster=0 - -collect_diagnostics() { - kubectl get cachebackends -A -o yaml >"$LOG_DIR/cachebackends.yaml" 2>&1 || true - kubectl -n "$SYSTEM_NAMESPACE" get all -o wide >"$LOG_DIR/system.txt" 2>&1 || true - kubectl -n "$SYSTEM_NAMESPACE" logs deployment/inference-cache-controller-manager --all-containers >"$LOG_DIR/controller.log" 2>&1 || true -} - -cleanup() { - rm -rf "$tmpdir" - if [ "$created_cluster" = "1" ] && [ "$KEEP_CLUSTER" != "1" ]; then - "$KIND" delete cluster --name "$KIND_CLUSTER" >/dev/null 2>&1 || true - fi -} - -on_exit() { - rc=$? - [ "$rc" -eq 0 ] || collect_diagnostics - cleanup - exit "$rc" -} -trap on_exit EXIT - -render_config() { - local source_dir="$1" destination="$2" controller_image="$3" server_image="$4" - cp -R "$source_dir/config" "$destination" - sed -i.bak \ - -e "/^- name: controller$/,/^- name: server$/ { s|^ newName: .*| newName: ${controller_image%:*}|; s|^ newTag: .*| newTag: ${controller_image##*:}|; }" \ - -e "/^- name: server$/,$ { s|^ newName: .*| newName: ${server_image%:*}|; s|^ newTag: .*| newTag: ${server_image##*:}|; }" \ - "$destination/default/kustomization.yaml" - rm -f "$destination/default/kustomization.yaml.bak" -} - -if "$KIND" get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER"; then - log "reusing kind cluster $KIND_CLUSTER" -else - "$KIND" create cluster --name "$KIND_CLUSTER" --wait 120s - created_cluster=1 -fi -kubectl config use-context "kind-$KIND_CLUSTER" >/dev/null - -log "installing cert-manager $CERT_MANAGER_VERSION" -kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/$CERT_MANAGER_VERSION/cert-manager.yaml" >/dev/null -kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout=180s - -phase5_src="$tmpdir/phase5-src" -mkdir -p "$phase5_src" -git -C "$REPO_ROOT" archive "$PHASE5_COMMIT" | tar -x -C "$phase5_src" - -log "building and installing Phase 5 at $PHASE5_COMMIT" -make -C "$phase5_src" image-build TAG="$PHASE5_TAG" REGISTRY="$REGISTRY" -"$KIND" load docker-image "$PHASE5_CONTROLLER_IMG" --name "$KIND_CLUSTER" -"$KIND" load docker-image "$PHASE5_SERVER_IMG" --name "$KIND_CLUSTER" -render_config "$phase5_src" "$tmpdir/phase5-config" "$PHASE5_CONTROLLER_IMG" "$PHASE5_SERVER_IMG" -kubectl apply -k "$tmpdir/phase5-config/default" >/dev/null -kubectl -n "$SYSTEM_NAMESPACE" wait --for=condition=Available --timeout="$READY_TIMEOUT" \ - deployment/inference-cache-controller-manager deployment/inference-cache-server - -kubectl create namespace "$SMOKE_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - >/dev/null -log "creating Phase 5 typed MP objects" -cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply -f - >/dev/null -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: phase5-host-only -spec: - runtime: VLLM - type: LMCache - engineSelector: - matchLabels: - app: phase5-engine - integration: - role: ReadWrite - lmCache: - topology: PodLocal - podLocal: - server: - image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 - port: 5555 - l1Capacity: 1Gi - maxWorkers: 1 - resources: - requests: {cpu: "1", memory: 2Gi} - limits: {memory: 2Gi} ---- -apiVersion: inferencecache.io/v1alpha1 -kind: CacheBackend -metadata: - name: phase5-managed-redis -spec: - runtime: SGLang - type: LMCache - engineSelector: - matchLabels: - app: phase5-sglang - integration: - role: ReadWrite - lmCache: - topology: PodLocal - podLocal: - server: - image: docker.io/lmcache/standalone@sha256:b813bf0bb616d1012b6a6edcbd4a44f1576dbbdaa857962e56d48b9f7c127d13 - port: 5555 - l1Capacity: 1Gi - maxWorkers: 1 - resources: - requests: {cpu: "1", memory: 2Gi} - limits: {memory: 2Gi} - remoteStorage: - provider: Redis - ownership: Managed - redis: {} -EOF - -host_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-host-only -o jsonpath='{.metadata.uid}')" -redis_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.metadata.uid}')" -[ -n "$host_uid" ] && [ -n "$redis_uid" ] || fail "Phase 5 objects were not persisted" - -log "upgrading CRDs and workloads to the current Phase 7 checkout" -make -C "$REPO_ROOT" image-build TAG="$TAG" REGISTRY="$REGISTRY" -"$KIND" load docker-image "$CURRENT_CONTROLLER_IMG" --name "$KIND_CLUSTER" -"$KIND" load docker-image "$CURRENT_SERVER_IMG" --name "$KIND_CLUSTER" -render_config "$REPO_ROOT" "$tmpdir/current-config" "$CURRENT_CONTROLLER_IMG" "$CURRENT_SERVER_IMG" -kubectl apply -k "$tmpdir/current-config/default" >/dev/null -kubectl -n "$SYSTEM_NAMESPACE" rollout status deployment/inference-cache-controller-manager --timeout="$READY_TIMEOUT" -kubectl -n "$SYSTEM_NAMESPACE" rollout status deployment/inference-cache-server --timeout="$READY_TIMEOUT" - -[ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-host-only -o jsonpath='{.metadata.uid}')" = "$host_uid" ] \ - || fail "host-only Phase 5 object was replaced during upgrade" -[ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.metadata.uid}')" = "$redis_uid" ] \ - || fail "managed-Redis Phase 5 object was replaced during upgrade" -for backend in phase5-host-only phase5-managed-redis; do - [ "$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend "$backend" -o jsonpath='{.spec.lmCache.topology}')" = "PodLocal" ] \ - || fail "$backend lost its typed MP topology" -done - -for _ in $(seq 1 60); do - endpoint="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend phase5-managed-redis -o jsonpath='{.status.remoteStorage.endpoint}' 2>/dev/null || true)" - [ "$endpoint" = "phase5-managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] && break - sleep 1 -done -[ "${endpoint:-}" = "phase5-managed-redis.$SMOKE_NAMESPACE.svc.cluster.local:6379" ] \ - || fail "managed Redis did not reconcile after upgrade" - -pod_json="$tmpdir/upgraded-admission.json" -cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply --dry-run=server -o json -f - >"$pod_json" -apiVersion: v1 -kind: Pod -metadata: - name: phase5-engine - labels: - app: phase5-engine -spec: - containers: - - name: vllm - image: busybox:1.36 - command: ["sh", "-c", "sleep 3600"] -EOF -grep -Fq 'lmcache-mp-server' "$pod_json" || fail "upgraded admission did not inject the MP server" -grep -Fq 'LMCacheMPConnector' "$pod_json" || fail "upgraded admission did not inject the MP connector" - -log "PASS: Phase 5 typed objects persisted and reconciled through the Phase 7 upgrade" diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go new file mode 100644 index 00000000..4caa0564 --- /dev/null +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go @@ -0,0 +1,535 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "crypto/sha256" + "fmt" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +const ( + lmCacheNodeLocalGateContainerName = "lmcache-node-local-gate" + lmCacheNodeLocalGateManagedEnv = "INFERENCECACHE_NODE_LOCAL_GATE" + lmCacheNodeLocalGateManagedValue = "true" + lmCacheNodeLocalShmVolumeName = "lmcache-node-shm" + lmCacheNodeLocalConfigVolumeName = "lmcache-node-config" + lmCacheNodeLocalConfigMountPath = "/var/run/inference-cache/lmcache-node" + lmCacheNodeLocalConfigFilePath = lmCacheNodeLocalConfigMountPath + "/client.yaml" + lmCacheNodeIPEnv = "INFERENCECACHE_NODE_IP" + lmCacheNodeLocalShmNamePrefix = "lmcache_l1_pool_inferencecache_" + posixShmNameMaxLength = 255 +) + +// RenderLMCacheNodeLocalServerPod renders the engine-neutral server for one +// node that already hosts a selected engine. Exact-node affinity leaves binding +// to the scheduler (so hostPort conflicts remain visible) and never steers the +// engine itself. The controller adds the owner reference; no Service is part of +// the NodeLocal MP contract. +func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding *backendadapter.Binding, nodeName string, source *corev1.Pod) (*corev1.Pod, error) { + if cache == nil || cache.Spec.LMCache == nil || cache.Spec.LMCache.Topology != cachev1alpha1.LMCacheTopologyNodeLocal || + cache.Spec.LMCache.NodeLocal == nil || cache.Spec.LMCache.NodeLocal.Server == nil { + return nil, fmt.Errorf("render LMCache NodeLocal server Pod: complete typed NodeLocal configuration is required") + } + if cache.UID == "" { + return nil, fmt.Errorf("render LMCache NodeLocal server Pod: CacheBackend UID is empty") + } + if strings.TrimSpace(nodeName) == "" || source == nil || source.Spec.NodeName != nodeName { + return nil, fmt.Errorf("render LMCache NodeLocal server Pod: a scheduled source engine on the target node is required") + } + server := cache.Spec.LMCache.NodeLocal.Server + scheduling := cache.Spec.LMCache.NodeLocal.Scheduling + if err := validateLMCacheNodeLocalServerConfig(server, effectiveLMCacheChunkSize(cache.Spec.LMCache)); err != nil { + return nil, err + } + l2Adapter, bindingEnv, err := renderLMCacheMPL2Binding(binding) + if err != nil { + return nil, err + } + l1GiB, err := quantityAsGiB(server.L1Capacity) + if err != nil { + return nil, err + } + + identity := lmCacheNodeLocalInstanceID(cache) + shmName, err := NodeLocalServerShmName(cache) + if err != nil { + return nil, err + } + args := []string{ + "server", + "--instance-id", identity, + "--shm-name", shmName, + "--host", "$(" + lmCacheNodeIPEnv + ")", + "--port", strconv.FormatInt(int64(server.Port), 10), + "--http-host", "$(" + lmCacheNodeIPEnv + ")", + "--http-port", strconv.FormatInt(int64(server.HTTPPort), 10), + "--chunk-size", strconv.FormatInt(int64(effectiveLMCacheChunkSize(cache.Spec.LMCache)), 10), + "--l1-size-gb", l1GiB, + "--eviction-policy", "LRU", + "--max-gpu-workers", strconv.FormatInt(int64(server.MaxGPUWorkers), 10), + "--max-cpu-workers", strconv.FormatInt(int64(server.MaxCPUWorkers), 10), + } + if l2Adapter != "" { + args = append(args, "--l2-adapter", l2Adapter) + } + env := []corev1.EnvVar{ + {Name: "NVIDIA_VISIBLE_DEVICES", Value: "all"}, + {Name: lmCacheMPServerManagedEnv, Value: lmCacheMPServerManagedValue}, + {Name: lmCacheNodeIPEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.hostIP"}}}, + } + for i := range bindingEnv { + env = UpsertEnv(env, bindingEnv[i]) + } + container := corev1.Container{ + Name: lmCacheMPServerContainerName, + Image: strings.TrimSpace(server.Image), + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"lmcache"}, + Args: args, + Env: env, + Resources: *server.Resources.DeepCopy(), + VolumeMounts: []corev1.VolumeMount{{ + Name: lmCacheNodeLocalShmVolumeName, MountPath: lmCacheMPShmMountPath, + }}, + Ports: []corev1.ContainerPort{ + {Name: lmCacheMPServerPortName, ContainerPort: server.Port, HostPort: server.Port, Protocol: corev1.ProtocolTCP}, + {Name: lmCacheMPHTTPPortName, ContainerPort: server.HTTPPort, HostPort: server.HTTPPort, Protocol: corev1.ProtocolTCP}, + }, + StartupProbe: lmCacheMPHTTPProbeForPort(lmCacheMPHTTPPortName, 3, 40), + ReadinessProbe: lmCacheMPHTTPProbeForPort(lmCacheMPHTTPPortName, 5, 3), + LivenessProbe: lmCacheMPHTTPProbeForPort(lmCacheMPHTTPPortName, 10, 3), + SecurityContext: lmCacheMPServerSecurityContext(nil), + } + + labels := map[string]string{ + "app.kubernetes.io/name": "lmcache-mp-server", + "app.kubernetes.io/managed-by": "inference-cache-controller", + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(cache.UID), + enginebinding.LabelLMCacheMPMetrics: enginebinding.LabelLMCacheMPMetricsEnabled, + } + annotations := map[string]string{ + enginebinding.AnnotationNodeLocalOwner: cache.Namespace + "/" + cache.Name, + enginebinding.AnnotationNodeLocalOwnerUID: string(cache.UID), + enginebinding.AnnotationNodeLocalGeneration: strconv.FormatInt(cache.Generation, 10), + enginebinding.AnnotationNodeLocalTargetNode: nodeName, + enginebinding.AnnotationNodeLocalShmName: shmName, + } + pathType := corev1.HostPathDirectory + noToken := false + enableServiceLinks := false + grace := int64(30) + tolerations := append([]corev1.Toleration(nil), source.Spec.Tolerations...) + imagePullSecrets := append([]corev1.LocalObjectReference(nil), source.Spec.ImagePullSecrets...) + priorityClassName := source.Spec.PriorityClassName + schedulerName := source.Spec.SchedulerName + runtimeClassName := copyStringPtr(source.Spec.RuntimeClassName) + serviceAccountName := "" + var podSecurityContext *corev1.PodSecurityContext + if scheduling != nil { + tolerations = mergeTolerations(tolerations, scheduling.Tolerations) + imagePullSecrets = mergeLocalObjectReferences(imagePullSecrets, scheduling.ImagePullSecrets) + serviceAccountName = scheduling.ServiceAccountName + podSecurityContext = scheduling.SecurityContext.DeepCopy() + if scheduling.PriorityClassName != "" { + priorityClassName = scheduling.PriorityClassName + } + if scheduling.SchedulerName != "" { + schedulerName = scheduling.SchedulerName + } + if scheduling.RuntimeClassName != nil { + runtimeClassName = copyStringPtr(scheduling.RuntimeClassName) + } + if scheduling.TerminationGracePeriodSeconds != nil { + grace = *scheduling.TerminationGracePeriodSeconds + } + } + podSpec := corev1.PodSpec{ + HostNetwork: true, + HostIPC: false, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + AutomountServiceAccountToken: &noToken, + EnableServiceLinks: &enableServiceLinks, + Affinity: exactNodeAffinity(nodeName), + Tolerations: tolerations, + ImagePullSecrets: imagePullSecrets, + ServiceAccountName: serviceAccountName, + SecurityContext: podSecurityContext, + PriorityClassName: priorityClassName, + SchedulerName: schedulerName, + RuntimeClassName: runtimeClassName, + TerminationGracePeriodSeconds: &grace, + RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{container}, + Volumes: []corev1.Volume{{ + Name: lmCacheNodeLocalShmVolumeName, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: lmCacheMPShmMountPath, Type: &pathType, + }}, + }}, + } + if podSpec.SchedulerName == "" { + podSpec.SchedulerName = "default-scheduler" + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: NodeLocalServerPodName(cache.Name, nodeName), Namespace: cache.Namespace, + Labels: copyStringMap(labels), Annotations: copyStringMap(annotations), + }, + Spec: podSpec, + }, nil +} + +// NodeLocalServerPodName returns the stable object name for one backend/node +// pair. It is exported for controller lifecycle reconciliation. +func NodeLocalServerPodName(backendName, nodeName string) string { + sum := sha256.Sum256([]byte(nodeName)) + suffix := fmt.Sprintf("%x", sum[:6]) + const maxName = 253 + maxPrefix := maxName - len(suffix) - 1 + prefix := strings.TrimRight(backendName, ".-") + if len(prefix) > maxPrefix { + prefix = strings.TrimRight(prefix[:maxPrefix], ".-") + } + return prefix + "-" + suffix +} + +// NodeLocalServerShmName returns the exact LMCache POSIX shared-memory object +// name owned by one CacheBackend UID. It intentionally excludes generation and +// node identity: replacements of the same backend reclaim their own stale +// object, while Kubernetes-assigned UIDs isolate delete/recreate lifecycles. +func NodeLocalServerShmName(cache *cachev1alpha1.CacheBackend) (string, error) { + if cache == nil || cache.UID == "" { + return "", fmt.Errorf("derive LMCache NodeLocal shm name: CacheBackend UID is empty") + } + uid := string(cache.UID) + for _, char := range uid { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || char == '-' || char == '_' || char == '.' { + continue + } + return "", fmt.Errorf("derive LMCache NodeLocal shm name: CacheBackend UID contains unsafe character %q", char) + } + name := lmCacheNodeLocalShmNamePrefix + uid + if len(name) > posixShmNameMaxLength { + return "", fmt.Errorf("derive LMCache NodeLocal shm name: %d-byte name exceeds POSIX limit %d", len(name), posixShmNameMaxLength) + } + return name, nil +} + +func exactNodeAffinity(nodeName string) *corev1.Affinity { + return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{nodeName}}}, + }}}, + }} +} + +func mergeTolerations(base, extra []corev1.Toleration) []corev1.Toleration { + out := append([]corev1.Toleration(nil), base...) + for i := range extra { + found := false + for j := range out { + if equality.Semantic.DeepEqual(out[j], extra[i]) { + found = true + break + } + } + if !found { + out = append(out, extra[i]) + } + } + return out +} + +func mergeLocalObjectReferences(base, extra []corev1.LocalObjectReference) []corev1.LocalObjectReference { + out := append([]corev1.LocalObjectReference(nil), base...) + seen := make(map[string]struct{}, len(out)) + for i := range out { + seen[out[i].Name] = struct{}{} + } + for i := range extra { + if _, found := seen[extra[i].Name]; found { + continue + } + seen[extra[i].Name] = struct{}{} + out = append(out, extra[i]) + } + return out +} + +func validateLMCacheNodeLocalServerConfig(server *cachev1alpha1.LMCacheNodeLocalServerSpec, chunkSize int32) error { + if server == nil { + return fmt.Errorf("render LMCache NodeLocal server: configuration is nil") + } + if strings.TrimSpace(server.Image) == "" { + return fmt.Errorf("render LMCache NodeLocal server: image is empty") + } + ports := []int32{server.Port, server.HTTPPort} + seen := map[int32]struct{}{} + for _, port := range ports { + if port < 1 || port > 65535 { + return fmt.Errorf("render LMCache NodeLocal server: port %d is outside 1-65535", port) + } + if _, duplicate := seen[port]; duplicate { + return fmt.Errorf("render LMCache NodeLocal server: host ports must be distinct") + } + seen[port] = struct{}{} + } + if chunkSize < 1 || server.L1Capacity.Sign() <= 0 || server.MaxGPUWorkers < 1 || server.MaxCPUWorkers < 1 { + return fmt.Errorf("render LMCache NodeLocal server: chunk size, L1 capacity, and worker counts must be positive") + } + return nil +} + +func lmCacheNodeLocalInstanceID(cache *cachev1alpha1.CacheBackend) string { + return fmt.Sprintf("%s/%s@%s#%d", cache.Namespace, cache.Name, cache.UID, cache.Generation) +} + +func lmCacheMPHTTPProbeForPort(portName string, period, failures int32) *corev1.Probe { + probe := lmCacheMPHTTPProbe(period, failures) + probe.HTTPGet.Port.StrVal = portName + return probe +} + +// renderLMCacheNodeLocalEngine injects the shared host /dev/shm mount and a +// startup gate. It deliberately leaves every engine placement field unchanged; +// the controller follows the engine onto its scheduled node. +func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName string, cache *cachev1alpha1.CacheBackend, writeClientConfig bool) (string, error) { + if pod == nil { + return "", fmt.Errorf("render LMCache NodeLocal engine: pod spec is nil") + } + engineIndex, err := EngineContainerIndexNamed(pod, engineContainerName) + if err != nil { + return "", err + } + if cache == nil || cache.Spec.LMCache == nil || cache.Spec.LMCache.NodeLocal == nil || + cache.Spec.LMCache.NodeLocal.Server == nil || cache.UID == "" { + return "", fmt.Errorf("render LMCache NodeLocal engine: complete typed NodeLocal configuration and CacheBackend UID are required") + } + server := cache.Spec.LMCache.NodeLocal.Server + if err := validateLMCacheNodeLocalServerConfig(server, effectiveLMCacheChunkSize(cache.Spec.LMCache)); err != nil { + return "", err + } + shmName, err := NodeLocalServerShmName(cache) + if err != nil { + return "", err + } + work := pod.DeepCopy() + engine := &work.Containers[engineIndex] + owned := lmCacheNodeLocalWireIsOurs(pod) + nodeIPEnv := corev1.EnvVar{ + Name: lmCacheNodeIPEnv, + ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "status.hostIP", + }}, + } + for i := range engine.Env { + if engine.Env[i].Name == lmCacheNodeIPEnv && !owned && !equality.Semantic.DeepEqual(engine.Env[i], nodeIPEnv) { + return "", fmt.Errorf("render LMCache NodeLocal engine: environment variable %q is reserved for the node address", lmCacheNodeIPEnv) + } + } + engine.Env = UpsertEnv(engine.Env, nodeIPEnv) + + shmMount := corev1.VolumeMount{Name: lmCacheNodeLocalShmVolumeName, MountPath: lmCacheMPShmMountPath} + if existing := mountAtPath(engine.VolumeMounts, lmCacheMPShmMountPath); existing != nil { + if err := checkNodeLocalHostShm(work.Volumes, *existing); err != nil { + return "", err + } + shmMount.Name = existing.Name + } else { + pathType := corev1.HostPathDirectory + work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ + Name: lmCacheNodeLocalShmVolumeName, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: lmCacheMPShmMountPath, Type: &pathType, + }}, + }, owned) + if err != nil { + return "", err + } + engine.VolumeMounts = upsertMountByName(engine.VolumeMounts, shmMount) + } + + configPath := "" + gateMounts := []corev1.VolumeMount{} + if writeClientConfig { + configPath = lmCacheNodeLocalConfigFilePath + if existing := mountAtPath(engine.VolumeMounts, lmCacheNodeLocalConfigMountPath); existing != nil && + !(owned && existing.Name == lmCacheNodeLocalConfigVolumeName) { + return "", fmt.Errorf("render LMCache NodeLocal engine: engine already mounts reserved config path %q", lmCacheNodeLocalConfigMountPath) + } + work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ + Name: lmCacheNodeLocalConfigVolumeName, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, owned) + if err != nil { + return "", err + } + configMount := corev1.VolumeMount{Name: lmCacheNodeLocalConfigVolumeName, MountPath: lmCacheNodeLocalConfigMountPath} + engine.VolumeMounts = upsertMountByName(engine.VolumeMounts, configMount) + gateMounts = append(gateMounts, configMount) + } + + gate := lmCacheNodeLocalGateContainer(cache, shmName, configPath, gateMounts) + work.InitContainers, err = adoptNodeLocalGate(work.InitContainers, gate, owned) + if err != nil { + return "", err + } + *pod = *work + return configPath, nil +} + +func lmCacheNodeLocalGateContainer(cache *cachev1alpha1.CacheBackend, shmName, configPath string, mounts []corev1.VolumeMount) corev1.Container { + server := cache.Spec.LMCache.NodeLocal.Server + const gateScript = `import json, os, time, urllib.request +ip = os.environ["INFERENCECACHE_NODE_IP"] +host = "[" + ip + "]" if ":" in ip else ip +base = "http://%s:%s" % (host, os.environ["EXPECTED_HTTP_PORT"]) +expected = { + "instance_id": os.environ["EXPECTED_INSTANCE_ID"], + "shm_name": os.environ["EXPECTED_SHM_NAME"], + "port": int(os.environ["EXPECTED_MP_PORT"]), + "chunk_size": int(os.environ["EXPECTED_CHUNK_SIZE"]), + "max_gpu_workers": int(os.environ["EXPECTED_MAX_GPU_WORKERS"]), + "max_cpu_workers": int(os.environ["EXPECTED_MAX_CPU_WORKERS"]), +} +while True: + try: + with urllib.request.urlopen(base + "/config", timeout=2) as response: + config = json.load(response) + mp = config.get("mp", {}) + http = config.get("http", {}) + for key, value in expected.items(): + if mp.get(key) != value: + raise RuntimeError("server config %s=%r, expected %r" % (key, mp.get(key), value)) + memory = config.get("storage_manager", {}).get("l1_manager_config", {}).get("memory_config", {}) + if memory.get("shm_name") != expected["shm_name"]: + raise RuntimeError("effective L1 shm_name=%r, expected %r" % (memory.get("shm_name"), expected["shm_name"])) + if http.get("http_port") != int(os.environ["EXPECTED_HTTP_PORT"]): + raise RuntimeError("server HTTP port does not match") + with urllib.request.urlopen(base + "/healthcheck", timeout=2) as response: + health = json.load(response) + if health.get("status") != "healthy": + raise RuntimeError("server health is %r" % health) + config_path = os.environ.get("CLIENT_CONFIG_PATH", "") + if config_path: + tmp = config_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as output: + # LMCache's SGLang MP adapter adds the tcp:// scheme itself; + # unlike vLLM's connector JSON, this YAML field is a bare host. + output.write('chunk_size: %s\nmp_host: "%s"\nmp_port: %s\n' % (expected["chunk_size"], ip, expected["port"])) + os.replace(tmp, config_path) + print("verified healthy same-node LMCache server " + expected["instance_id"], flush=True) + break + except Exception as exc: + print("waiting for ownership-verified same-node LMCache server: %s" % exc, flush=True) + time.sleep(2) +` + return corev1.Container{ + Name: lmCacheNodeLocalGateContainerName, + Image: strings.TrimSpace(server.Image), + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"python3", "-c"}, + Args: []string{gateScript}, + Env: []corev1.EnvVar{ + {Name: lmCacheNodeLocalGateManagedEnv, Value: lmCacheNodeLocalGateManagedValue}, + {Name: lmCacheNodeIPEnv, ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "status.hostIP"}}}, + {Name: "EXPECTED_INSTANCE_ID", Value: lmCacheNodeLocalInstanceID(cache)}, + {Name: "EXPECTED_SHM_NAME", Value: shmName}, + {Name: "EXPECTED_MP_PORT", Value: strconv.FormatInt(int64(server.Port), 10)}, + {Name: "EXPECTED_HTTP_PORT", Value: strconv.FormatInt(int64(server.HTTPPort), 10)}, + {Name: "EXPECTED_CHUNK_SIZE", Value: strconv.FormatInt(int64(effectiveLMCacheChunkSize(cache.Spec.LMCache)), 10)}, + {Name: "EXPECTED_MAX_GPU_WORKERS", Value: strconv.FormatInt(int64(server.MaxGPUWorkers), 10)}, + {Name: "EXPECTED_MAX_CPU_WORKERS", Value: strconv.FormatInt(int64(server.MaxCPUWorkers), 10)}, + {Name: "CLIENT_CONFIG_PATH", Value: configPath}, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("32Mi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}, + }, + VolumeMounts: mounts, + SecurityContext: lmCacheMPServerSecurityContext(nil), + } +} + +func checkNodeLocalHostShm(volumes []corev1.Volume, mount corev1.VolumeMount) error { + if mount.ReadOnly || mount.SubPath != "" || mount.SubPathExpr != "" { + return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm must be a writable whole-volume hostPath mount") + } + for i := range volumes { + if volumes[i].Name != mount.Name { + continue + } + hostPath := volumes[i].HostPath + if hostPath == nil || hostPath.Path != lmCacheMPShmMountPath { + return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm must use hostPath /dev/shm for cross-Pod CUDA IPC") + } + return nil + } + return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm mount references missing volume %q", mount.Name) +} + +func lmCacheNodeLocalWireIsOurs(pod *corev1.PodSpec) bool { + if pod == nil { + return false + } + gate := findContainerByName(pod.InitContainers, lmCacheNodeLocalGateContainerName) + if gate == nil { + return false + } + for i := range gate.Env { + if gate.Env[i].Name == lmCacheNodeLocalGateManagedEnv && gate.Env[i].Value == lmCacheNodeLocalGateManagedValue { + return true + } + } + return false +} + +func adoptNodeLocalGate(containers []corev1.Container, want corev1.Container, owned bool) ([]corev1.Container, error) { + for i := range containers { + if containers[i].Name != want.Name { + continue + } + if !owned { + return nil, fmt.Errorf("render LMCache NodeLocal engine: init container name %q is reserved for the ownership-verification gate", want.Name) + } + containers[i] = want + return containers, nil + } + return append(containers, want), nil +} + +func copyStringMap(in map[string]string) map[string]string { + if in == nil { + return nil + } + out := make(map[string]string, len(in)) + for key, value := range in { + out[key] = value + } + return out +} + +func copyStringPtr(in *string) *string { + if in == nil { + return nil + } + out := new(string) + *out = *in + return out +} diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go new file mode 100644 index 00000000..9041fc7f --- /dev/null +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go @@ -0,0 +1,464 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package runtime + +import ( + "reflect" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +func newNodeLocalBackend(runtime cachev1alpha1.CacheBackendRuntime) *cachev1alpha1.CacheBackend { + chunk := int32(256) + runtimeClass := "nvidia" + return &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "node-cache", Namespace: "team-a", UID: types.UID("11111111-2222-3333-4444-555555555555"), Generation: 7}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: runtime, + Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyNodeLocal, + ChunkSizeTokens: &chunk, + NodeLocal: &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: testLMCacheServerImage, + Port: 6555, + HTTPPort: 18080, + L1Capacity: resource.MustParse("8Gi"), + MaxGPUWorkers: 4, + MaxCPUWorkers: 8, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("9Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("10Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{ + RuntimeClassName: &runtimeClass, + }, + }, + }, + Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + "app": "engine", + }}, + }, + } +} + +func nodeLocalSourceEngine() *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "engine-a", Namespace: "team-a"}, Spec: corev1.PodSpec{ + NodeName: "gpu-node-a", Tolerations: []corev1.Toleration{{Key: "nvidia.com/gpu", Operator: corev1.TolerationOpExists}}, + Containers: []corev1.Container{{Name: EngineContainerName}}, + }} +} + +func TestRenderLMCacheNodeLocalServerPod(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + serverPod, err := RenderLMCacheNodeLocalServerPod(cache, nil, "gpu-node-a", nodeLocalSourceEngine()) + if err != nil { + t.Fatalf("RenderLMCacheNodeLocalServerPod: %v", err) + } + if serverPod.Name != NodeLocalServerPodName(cache.Name, "gpu-node-a") || serverPod.Namespace != cache.Namespace { + t.Fatalf("server Pod identity = %s/%s", serverPod.Namespace, serverPod.Name) + } + pod := serverPod.Spec + if !pod.HostNetwork || pod.HostIPC || pod.DNSPolicy != corev1.DNSClusterFirstWithHostNet { + t.Fatalf("host boundary = hostNetwork:%v hostIPC:%v dns:%s", pod.HostNetwork, pod.HostIPC, pod.DNSPolicy) + } + if pod.RuntimeClassName == nil || *pod.RuntimeClassName != "nvidia" || len(pod.NodeSelector) != 0 || + pod.Affinity == nil || pod.Affinity.NodeAffinity == nil { + t.Fatalf("placement = runtimeClass:%v nodeSelector:%v affinity:%+v", pod.RuntimeClassName, pod.NodeSelector, pod.Affinity) + } + required := pod.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + if required == nil || len(required.NodeSelectorTerms) != 1 || len(required.NodeSelectorTerms[0].MatchFields) != 1 || + required.NodeSelectorTerms[0].MatchFields[0].Key != "metadata.name" || + !reflect.DeepEqual(required.NodeSelectorTerms[0].MatchFields[0].Values, []string{"gpu-node-a"}) { + t.Fatalf("exact-node scheduler affinity = %+v", required) + } + if len(pod.Volumes) != 1 || pod.Volumes[0].HostPath == nil || pod.Volumes[0].HostPath.Path != "/dev/shm" { + t.Fatalf("volumes = %+v, want host /dev/shm", pod.Volumes) + } + if len(pod.Containers) != 1 { + t.Fatalf("containers = %d, want one", len(pod.Containers)) + } + server := pod.Containers[0] + if len(server.Ports) != 2 { + t.Fatalf("ports = %+v", server.Ports) + } + for _, port := range server.Ports { + if port.HostPort != port.ContainerPort || port.HostPort == 0 { + t.Fatalf("port does not declare matching hostPort: %+v", port) + } + } + args := strings.Join(server.Args, " ") + for _, want := range []string{ + "--instance-id team-a/node-cache@11111111-2222-3333-4444-555555555555#7", + "--shm-name lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555", + "--host $(INFERENCECACHE_NODE_IP)", "--http-port 18080", + "--max-gpu-workers 4", "--max-cpu-workers 8", + } { + if !strings.Contains(args, want) { + t.Fatalf("server args %q missing %q", args, want) + } + } + if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID]; got != string(cache.UID) { + t.Fatalf("owner UID annotation = %q", got) + } + if got := serverPod.Labels[enginebinding.LabelCacheBackendUID]; got != string(cache.UID) { + t.Fatalf("owner UID label = %q", got) + } + if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalTargetNode]; got != "gpu-node-a" { + t.Fatalf("target-node annotation = %q", got) + } + if got := serverPod.Annotations[enginebinding.AnnotationNodeLocalShmName]; got != "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" { + t.Fatalf("shared-memory annotation = %q", got) + } +} + +func TestRenderLMCacheNodeLocalServerPodWithRedisL2(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + pod, err := RenderLMCacheNodeLocalServerPod(cache, &backendadapter.Binding{ + Protocol: backendadapter.ProtocolRESP, + Endpoint: "redis.team-a.svc.cluster.local:6379", + }, "gpu-node-a", nodeLocalSourceEngine()) + if err != nil { + t.Fatalf("RenderLMCacheNodeLocalServerPod: %v", err) + } + server := pod.Spec.Containers[0] + args := strings.Join(server.Args, " ") + if !strings.Contains(args, "--l2-adapter") || !strings.Contains(args, "redis.team-a.svc.cluster.local") || !strings.Contains(args, `"port":6379`) { + t.Fatalf("server args = %v, want typed Redis L2 adapter", server.Args) + } +} + +func TestRenderLMCacheNodeLocalServerPodSchedulingOverridesAndMerges(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + grace := int64(9) + runtimeClass := "cache-nvidia" + runAsNonRoot := true + cache.Spec.LMCache.NodeLocal.Scheduling = &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{ + Tolerations: []corev1.Toleration{ + {Key: "source", Operator: corev1.TolerationOpExists}, + {Key: "cache", Operator: corev1.TolerationOpExists}, + }, + ImagePullSecrets: []corev1.LocalObjectReference{{Name: "source-pull"}, {Name: "cache-pull"}}, + ServiceAccountName: "cache-server", + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &runAsNonRoot}, + PriorityClassName: "cache-priority", + SchedulerName: "cache-scheduler", + RuntimeClassName: &runtimeClass, + TerminationGracePeriodSeconds: &grace, + } + source := nodeLocalSourceEngine() + source.Spec.Tolerations = []corev1.Toleration{{Key: "source", Operator: corev1.TolerationOpExists}} + source.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "source-pull"}} + source.Spec.PriorityClassName = "engine-priority" + source.Spec.SchedulerName = "engine-scheduler" + + server, err := RenderLMCacheNodeLocalServerPod(cache, nil, "gpu-node-a", source) + if err != nil { + t.Fatalf("RenderLMCacheNodeLocalServerPod: %v", err) + } + got := server.Spec + if len(got.Tolerations) != 2 || len(got.ImagePullSecrets) != 2 { + t.Fatalf("merged scheduling = tolerations:%+v pullSecrets:%+v", got.Tolerations, got.ImagePullSecrets) + } + if got.ServiceAccountName != "cache-server" || got.SecurityContext == nil || got.SecurityContext.RunAsNonRoot == nil || !*got.SecurityContext.RunAsNonRoot || + got.PriorityClassName != "cache-priority" || got.SchedulerName != "cache-scheduler" || got.RuntimeClassName == nil || *got.RuntimeClassName != runtimeClass || + got.TerminationGracePeriodSeconds == nil || *got.TerminationGracePeriodSeconds != grace { + t.Fatalf("server scheduling overrides = %+v", got) + } +} + +func TestNodeLocalServerPodNameIsStableAndBounded(t *testing.T) { + nameA := NodeLocalServerPodName(strings.Repeat("cache", 70), "gpu-node-a") + nameB := NodeLocalServerPodName(strings.Repeat("cache", 70), "gpu-node-b") + if len(nameA) > 253 || nameA == nameB || nameA != NodeLocalServerPodName(strings.Repeat("cache", 70), "gpu-node-a") { + t.Fatalf("server names are not stable, bounded, and node-distinct: %q %q", nameA, nameB) + } +} + +func TestNodeLocalServerShmNameIsStableAndUIDScoped(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + want := "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" + got, err := NodeLocalServerShmName(cache) + if err != nil || got != want { + t.Fatalf("NodeLocalServerShmName = %q, %v; want %q", got, err, want) + } + cache.Name = "renamed" + cache.Namespace = "other" + cache.Generation++ + stable, err := NodeLocalServerShmName(cache) + if err != nil || stable != want { + t.Fatalf("same-UID replacement shm name = %q, %v; want %q", stable, err, want) + } + cache.UID = types.UID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + distinct, err := NodeLocalServerShmName(cache) + if err != nil || distinct == want { + t.Fatalf("different-UID shm name = %q, %v; must differ from %q", distinct, err, want) + } +} + +func TestNodeLocalServerShmNameRejectsUnsafeUID(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + cache.UID = types.UID("unsafe/uid") + if _, err := NodeLocalServerShmName(cache); err == nil || !strings.Contains(err.Error(), "unsafe character") { + t.Fatalf("unsafe UID error = %v", err) + } + cache.UID = types.UID(strings.Repeat("a", posixShmNameMaxLength)) + if _, err := NodeLocalServerShmName(cache); err == nil || !strings.Contains(err.Error(), "exceeds POSIX limit") { + t.Fatalf("oversized UID error = %v", err) + } +} + +func TestVLLMNodeLocalEngineInjection(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + pod := newVLLMMPEnginePod("--tensor-parallel-size", "1") + pod.Spec.NodeSelector = map[string]string{"inference-system.io/pool": "owned"} + pod.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{{Weight: 1}}}} + wantSelector := map[string]string{"inference-system.io/pool": "owned"} + wantAffinity := pod.Spec.Affinity.DeepCopy() + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + if err := adapter.InjectEngineConfig(&pod.Spec, nil, cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + if pod.Spec.HostNetwork || pod.Spec.HostIPC { + t.Fatalf("engine unexpectedly entered host network/IPC: %+v", pod.Spec) + } + if !reflect.DeepEqual(pod.Spec.NodeSelector, wantSelector) || !reflect.DeepEqual(pod.Spec.Affinity, wantAffinity) { + t.Fatalf("engine-owned placement was mutated: selector=%v affinity=%+v", pod.Spec.NodeSelector, pod.Spec.Affinity) + } + if len(pod.Spec.InitContainers) != 1 || pod.Spec.InitContainers[0].Name != lmCacheNodeLocalGateContainerName { + t.Fatalf("init containers = %+v", pod.Spec.InitContainers) + } + if findContainerByName(pod.Spec.InitContainers, lmCacheMPServerContainerName) != nil { + t.Fatal("NodeLocal engine received a PodLocal native sidecar") + } + engine := pod.Spec.Containers[0] + if !envHasFieldRef(engine.Env, lmCacheNodeIPEnv, "status.hostIP") { + t.Fatalf("engine node IP env = %+v", engine.Env) + } + joined := strings.Join(engine.Args, " ") + if !strings.Contains(joined, `tcp://$(INFERENCECACHE_NODE_IP)`) || !strings.Contains(joined, `"lmcache.mp.port":"6555"`) { + t.Fatalf("vLLM args do not carry node-derived endpoint: %s", joined) + } + if mountAtPath(engine.VolumeMounts, "/dev/shm") == nil { + t.Fatalf("engine mounts = %+v, want host /dev/shm", engine.VolumeMounts) + } + gate := pod.Spec.InitContainers[0] + if !strings.Contains(gate.Args[0], "/config") || !strings.Contains(gate.Args[0], "EXPECTED_INSTANCE_ID") || + !strings.Contains(gate.Args[0], `memory.get("shm_name")`) { + t.Fatalf("gate script does not verify live server identity: %q", gate.Args[0]) + } + if got, found := lookupEnv(gate.Env, "EXPECTED_SHM_NAME"); !found || got != "lmcache_l1_pool_inferencecache_11111111-2222-3333-4444-555555555555" { + t.Fatalf("gate EXPECTED_SHM_NAME = %q, found=%v", got, found) + } + + before := pod.Spec.DeepCopy() + if err := adapter.InjectEngineConfig(&pod.Spec, nil, cache); err != nil { + t.Fatalf("idempotent InjectEngineConfig: %v", err) + } + if !reflect.DeepEqual(before, &pod.Spec) { + t.Fatal("NodeLocal vLLM injection is not idempotent") + } +} + +func TestSGLangNodeLocalEngineInjectionWritesEngineSpecificConfig(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeSGLang) + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Image: "sglang:lmcache", Args: []string{"--page-size", "64"}, + }}}} + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) + if err := adapter.InjectEngineConfig(&pod.Spec, nil, cache); err != nil { + t.Fatalf("InjectEngineConfig: %v", err) + } + engine := pod.Spec.Containers[0] + if !hasArg(engine.Args, SGLangEnableLMCacheArg) || !strings.Contains(strings.Join(engine.Args, " "), lmCacheNodeLocalConfigFilePath) { + t.Fatalf("SGLang args = %v", engine.Args) + } + gate := findContainerByName(pod.Spec.InitContainers, lmCacheNodeLocalGateContainerName) + if gate == nil || !strings.Contains(gate.Args[0], `mp_host: "%s"`) || strings.Contains(gate.Args[0], `mp_host: "tcp://%s"`) { + t.Fatalf("SGLang config-writing gate = %+v", gate) + } +} + +func TestNodeLocalEnginePlacementIsPreserved(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + cache.Spec.LMCache.NodeLocal.Scheduling = nil + pod := newVLLMMPEnginePod() + pod.Spec.NodeSelector = map[string]string{"inferencecache.io/lmcache-mp": "false"} + if err := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).InjectEngineConfig(&pod.Spec, nil, cache); err != nil { + t.Fatalf("inference-owned selector should not conflict: %v", err) + } + if !reflect.DeepEqual(pod.Spec.NodeSelector, map[string]string{"inferencecache.io/lmcache-mp": "false"}) { + t.Fatalf("engine nodeSelector mutated: %v", pod.Spec.NodeSelector) + } +} + +func TestSGLangNodeLocalEngineInjectionDoesNotRequireSchedulingOverrides(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeSGLang) + cache.Spec.LMCache.NodeLocal.Scheduling = nil + pod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Image: "sglang:lmcache", Args: []string{"--page-size", "64"}, + }}}} + if err := NewSGLangLMCacheAdapter(SubscriberConfig{}).InjectEngineConfig(&pod.Spec, nil, cache); err != nil { + t.Fatalf("optional scheduling overrides should not be required: %v", err) + } +} + +func TestRenderLMCacheNodeLocalServerPodRejectsInvalidContracts(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend) + want string + }{ + {name: "missing UID", mutate: func(cache *cachev1alpha1.CacheBackend) { cache.UID = "" }, want: "UID is empty"}, + {name: "empty image", mutate: func(cache *cachev1alpha1.CacheBackend) { cache.Spec.LMCache.NodeLocal.Server.Image = " " }, want: "image is empty"}, + {name: "invalid port", mutate: func(cache *cachev1alpha1.CacheBackend) { cache.Spec.LMCache.NodeLocal.Server.Port = 0 }, want: "outside 1-65535"}, + {name: "duplicate host ports", mutate: func(cache *cachev1alpha1.CacheBackend) { + cache.Spec.LMCache.NodeLocal.Server.HTTPPort = cache.Spec.LMCache.NodeLocal.Server.Port + }, want: "host ports must be distinct"}, + {name: "zero worker count", mutate: func(cache *cachev1alpha1.CacheBackend) { cache.Spec.LMCache.NodeLocal.Server.MaxGPUWorkers = 0 }, want: "must be positive"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + tt.mutate(cache) + _, err := RenderLMCacheNodeLocalServerPod(cache, nil, "gpu-node-a", nodeLocalSourceEngine()) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want substring %q", err, tt.want) + } + }) + } + if _, err := RenderLMCacheNodeLocalServerPod(nil, nil, "gpu-node-a", nodeLocalSourceEngine()); err == nil || !strings.Contains(err.Error(), "complete typed") { + t.Fatalf("nil CacheBackend error = %v", err) + } + if _, err := RenderLMCacheNodeLocalServerPod(newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM), nil, "", nodeLocalSourceEngine()); err == nil || !strings.Contains(err.Error(), "scheduled source") { + t.Fatalf("missing target node error = %v", err) + } +} + +func TestNodeLocalEngineInjectionRejectsReservedWireCollisionsAtomically(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PodSpec) + want string + }{ + { + name: "node IP env", + mutate: func(pod *corev1.PodSpec) { + pod.Containers[0].Env = append(pod.Containers[0].Env, corev1.EnvVar{Name: lmCacheNodeIPEnv, Value: "192.0.2.1"}) + }, + want: "environment variable", + }, + { + name: "gate name", + mutate: func(pod *corev1.PodSpec) { + pod.InitContainers = append(pod.InitContainers, corev1.Container{Name: lmCacheNodeLocalGateContainerName, Image: "user/gate:latest"}) + }, + want: "init container name", + }, + { + name: "read-only shared memory", + mutate: func(pod *corev1.PodSpec) { + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev/shm"}}}) + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm", ReadOnly: true}) + }, + want: "writable whole-volume", + }, + { + name: "non-host shared memory", + mutate: func(pod *corev1.PodSpec) { + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"}) + }, + want: "must use hostPath", + }, + { + name: "missing shared memory volume", + mutate: func(pod *corev1.PodSpec) { + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "missing", MountPath: "/dev/shm"}) + }, + want: "references missing volume", + }, + } + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + pod := newVLLMMPEnginePod() + tt.mutate(&pod.Spec) + before := pod.Spec.DeepCopy() + err := adapter.InjectEngineConfig(&pod.Spec, nil, cache) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want substring %q", err, tt.want) + } + if !reflect.DeepEqual(before, &pod.Spec) { + t.Fatal("failed NodeLocal injection partially mutated PodSpec") + } + }) + } +} + +func TestRenderLMCacheNodeLocalEngineRejectsMissingInputs(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + if _, err := renderLMCacheNodeLocalEngine(nil, EngineContainerName, cache, false); err == nil || !strings.Contains(err.Error(), "pod spec is nil") { + t.Fatalf("nil pod error = %v", err) + } + pod := newVLLMMPEnginePod().Spec + cache.UID = "" + if _, err := renderLMCacheNodeLocalEngine(&pod, EngineContainerName, cache, false); err == nil || !strings.Contains(err.Error(), "complete typed") { + t.Fatalf("missing backend identity error = %v", err) + } +} + +func TestNodeLocalAdaptersValidateTopologyContract(t *testing.T) { + vllmCache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + vllmPod := newVLLMMPEnginePod("--tensor-parallel-size", "1") + vllm := vllmLMCacheMPAdapter{} + if !vllm.Supports("vllm", vllmCache) { + t.Fatal("vLLM adapter does not advertise typed NodeLocal support") + } + if err := vllm.ValidateMPEnginePod(vllmPod, vllmCache); err != nil { + t.Fatalf("validate vLLM NodeLocal engine: %v", err) + } + vllmCache.Spec.LMCache.NodeLocal.Scheduling = nil + if err := vllm.ValidateMPEnginePod(vllmPod, vllmCache); err != nil { + t.Fatalf("vLLM optional scheduling rejected: %v", err) + } + vllmCache.Spec.LMCache.Topology = "Unsupported" + if err := vllm.ValidateMPEnginePod(vllmPod, vllmCache); err == nil || !strings.Contains(err.Error(), "not implemented") { + t.Fatalf("vLLM unsupported topology error = %v", err) + } + + sglangCache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeSGLang) + sglangPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{ + Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}, + }}}} + sglang := sglangLMCacheAdapter{} + if !sglang.Supports("sglang", sglangCache) { + t.Fatal("SGLang adapter does not advertise typed NodeLocal support") + } + if err := sglang.ValidateMPEnginePod(sglangPod, sglangCache); err != nil { + t.Fatalf("validate SGLang NodeLocal engine: %v", err) + } + sglangCache.Spec.LMCache.NodeLocal.Server = nil + if err := sglang.ValidateMPEnginePod(sglangPod, sglangCache); err == nil || !strings.Contains(err.Error(), "server configuration") { + t.Fatalf("SGLang missing server error = %v", err) + } + sglangCache.Spec.LMCache.Topology = "Unsupported" + if err := sglang.ValidateMPEnginePod(sglangPod, sglangCache); err == nil || !strings.Contains(err.Error(), "not implemented") { + t.Fatalf("SGLang unsupported topology error = %v", err) + } +} diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go index 2b9d8397..23de7618 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer.go @@ -58,7 +58,7 @@ type lmCacheMPServerConfig struct { // WriteClientConfig asks the native sidecar to create the generic LMCache // MP client YAML shared with the engine. SGLang consumes this file. vLLM's - // connector consumes JSON and will set this false in Phase 4. + // connector consumes JSON and sets this false. WriteClientConfig bool } diff --git a/internal/adapters/builtin/runtime/sglang_lmcache.go b/internal/adapters/builtin/runtime/sglang_lmcache.go index a4a8b9e6..2fb01646 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache.go @@ -54,10 +54,10 @@ const ( // sglangLMCacheAdapter wires SGLang engine pods to LMCache for the (SGLang, LMCache) // pair. SGLang drives LMCache in MULTIPROCESS (MP) mode: // -// - Typed PodLocal objects use the shared CacheBackend-configured MP-server native -// sidecar + a config file (mp_host/mp_port) the engine reads via -// --lmcache-config-file. A nil binding is L1-only; an optional RESP binding -// offloads to independently selected Redis storage. +// - Typed PodLocal objects use the shared CacheBackend-configured MP-server +// native sidecar. Typed NodeLocal objects use an on-demand same-node server +// Pod and an ownership-verifying startup gate. Both render the config file +// (mp_host/mp_port) read through --lmcache-config-file. // - It turns LMCache on with // --enable-lmcache + LMCACHE_USE_EXPERIMENTAL (not vLLM's --kv-transfer-config) // and does not inject any IP-connector environment. @@ -88,7 +88,9 @@ func (sglangLMCacheAdapter) Supports(runtime runtimeadapter.RuntimeID, cache *ca return false } return cache.Spec.IsEventsOnly() || - (cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal) + (cache.Spec.LMCache != nil && + (cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal || + cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyNodeLocal)) } // SupportedPairs lets the registry surface this adapter's canonical pair in the @@ -108,7 +110,7 @@ func (sglangLMCacheAdapter) SupportsBinding(binding *backendadapter.Binding) boo // InjectEngineConfig renders SGLang's LMCache MP-mode launch surface from a // host-only nil binding or a RESP binding for Redis L2 storage. func (a sglangLMCacheAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { - if err := injectSGLangLMCachePodLocal(pod, binding, cache); err != nil { + if err := injectSGLangLMCacheMP(pod, binding, cache); err != nil { return err } return ensureSGLangMetricsForSubscriber(pod, cache, a.subscriber) @@ -139,12 +141,17 @@ func (sglangLMCacheAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a if cache == nil || cache.Spec.LMCache == nil { return fmt.Errorf("SGLang LMCache MP CacheBackend configuration is missing") } - if cache.Spec.LMCache.Topology != cachev1alpha1.LMCacheTopologyPodLocal { - return fmt.Errorf("SGLang LMCache MP topology %q is not implemented; want %q", - cache.Spec.LMCache.Topology, cachev1alpha1.LMCacheTopologyPodLocal) - } - if cache.Spec.LMCache.PodLocal == nil || cache.Spec.LMCache.PodLocal.Server == nil { - return fmt.Errorf("SGLang LMCache PodLocal server configuration is missing") + switch cache.Spec.LMCache.Topology { + case cachev1alpha1.LMCacheTopologyPodLocal: + if cache.Spec.LMCache.PodLocal == nil || cache.Spec.LMCache.PodLocal.Server == nil { + return fmt.Errorf("SGLang LMCache PodLocal server configuration is missing") + } + case cachev1alpha1.LMCacheTopologyNodeLocal: + if cache.Spec.LMCache.NodeLocal == nil || cache.Spec.LMCache.NodeLocal.Server == nil { + return fmt.Errorf("SGLang LMCache NodeLocal server configuration is missing") + } + default: + return fmt.Errorf("SGLang LMCache MP topology %q is not implemented", cache.Spec.LMCache.Topology) } engineIndex, err := EngineContainerIndexNamed(&pod.Spec, SGLangEngineContainerName) if err != nil { @@ -195,31 +202,46 @@ func effectiveLMCacheChunkSize(spec *cachev1alpha1.LMCacheEngineSpec) int32 { return 256 } -func injectSGLangLMCachePodLocal(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { +func injectSGLangLMCacheMP(pod *corev1.PodSpec, binding *backendadapter.Binding, cache *cachev1alpha1.CacheBackend) error { if err := validateInjectPodCacheInputs(pod, cache, "engine"); err != nil { return err } lm := cache.Spec.LMCache - if lm == nil || lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal || lm.PodLocal == nil || lm.PodLocal.Server == nil { - return fmt.Errorf("inject SGLang LMCache MP: typed PodLocal server configuration is required") + if lm == nil { + return fmt.Errorf("inject SGLang LMCache MP: typed server configuration is required") } - server := lm.PodLocal.Server chunkSize := effectiveLMCacheChunkSize(lm) // Compose the common server and SGLang launch surface on one copy. Although // the post-render SGLang upserts cannot fail, keeping one commit point makes // the adapter's atomicity contract explicit and future-proof. work := pod.DeepCopy() - configPath, err := renderLMCachePodLocalServer(work, SGLangEngineContainerName, lmCacheMPServerConfig{ - Image: server.Image, - Port: server.Port, - ChunkSizeTokens: chunkSize, - L1Capacity: server.L1Capacity, - MaxWorkers: server.MaxWorkers, - Resources: server.Resources, - Binding: binding, - WriteClientConfig: true, - }) + var configPath string + var err error + switch lm.Topology { + case cachev1alpha1.LMCacheTopologyPodLocal: + if lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("inject SGLang LMCache MP: typed PodLocal server configuration is required") + } + server := lm.PodLocal.Server + configPath, err = renderLMCachePodLocalServer(work, SGLangEngineContainerName, lmCacheMPServerConfig{ + Image: server.Image, + Port: server.Port, + ChunkSizeTokens: chunkSize, + L1Capacity: server.L1Capacity, + MaxWorkers: server.MaxWorkers, + Resources: server.Resources, + Binding: binding, + WriteClientConfig: true, + }) + case cachev1alpha1.LMCacheTopologyNodeLocal: + if lm.NodeLocal == nil || lm.NodeLocal.Server == nil { + return fmt.Errorf("inject SGLang LMCache MP: typed NodeLocal server configuration is required") + } + configPath, err = renderLMCacheNodeLocalEngine(work, SGLangEngineContainerName, cache, true) + default: + return fmt.Errorf("inject SGLang LMCache MP: topology %q is not implemented", lm.Topology) + } if err != nil { return err } diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go index 0a16017a..1b74458a 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp.go @@ -33,7 +33,7 @@ type vllmLMCacheMPAdapter struct { subscriber SubscriberConfig } -// NewVLLMLMCacheMPAdapter returns the typed PodLocal vLLM adapter. +// NewVLLMLMCacheMPAdapter returns the typed PodLocal/NodeLocal vLLM adapter. func NewVLLMLMCacheMPAdapter(subscriber SubscriberConfig) runtimeadapter.KVCacheRuntimeAdapter { return vllmLMCacheMPAdapter{subscriber: subscriber} } @@ -48,7 +48,9 @@ func (vllmLMCacheMPAdapter) Supports(runtime runtimeadapter.RuntimeID, cache *ca return false } return cache.Spec.IsEventsOnly() || - (cache.Spec.LMCache != nil && cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal) + (cache.Spec.LMCache != nil && + (cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal || + cache.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyNodeLocal)) } func (vllmLMCacheMPAdapter) SupportsBinding(binding *backendadapter.Binding) bool { @@ -68,12 +70,17 @@ func (vllmLMCacheMPAdapter) ValidateMPEnginePod(pod *corev1.Pod, cache *cachev1a return fmt.Errorf("vLLM LMCache MP CacheBackend configuration is missing") } lm := cache.Spec.LMCache - if lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal { - return fmt.Errorf("vLLM LMCache MP topology %q is not implemented; want %q", - lm.Topology, cachev1alpha1.LMCacheTopologyPodLocal) - } - if lm.PodLocal == nil || lm.PodLocal.Server == nil { - return fmt.Errorf("vLLM LMCache PodLocal server configuration is missing") + switch lm.Topology { + case cachev1alpha1.LMCacheTopologyPodLocal: + if lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("vLLM LMCache PodLocal server configuration is missing") + } + case cachev1alpha1.LMCacheTopologyNodeLocal: + if lm.NodeLocal == nil || lm.NodeLocal.Server == nil { + return fmt.Errorf("vLLM LMCache NodeLocal server configuration is missing") + } + default: + return fmt.Errorf("vLLM LMCache MP topology %q is not implemented", lm.Topology) } engineIndex, err := EngineContainerIndexNamed(&pod.Spec, EngineContainerName) if err != nil { @@ -190,6 +197,10 @@ type vllmMPConnectorExtraConfig struct { } func vllmMPKVTransferConfigJSON(role cachev1alpha1.CacheBackendIntegrationRole, port int32) (string, error) { + return vllmMPKVTransferConfigJSONForHost(role, "tcp://127.0.0.1", port) +} + +func vllmMPKVTransferConfigJSONForHost(role cachev1alpha1.CacheBackendIntegrationRole, host string, port int32) (string, error) { kvRole := "" switch role { case cachev1alpha1.CacheBackendIntegrationRoleReadOnly: @@ -206,7 +217,7 @@ func vllmMPKVTransferConfigJSON(role cachev1alpha1.CacheBackendIntegrationRole, ConnectorModule: vllmLMCacheMPConnectorModulePath, Role: kvRole, ExtraConfig: vllmMPConnectorExtraConfig{ - Host: "tcp://127.0.0.1", + Host: host, Port: strconv.FormatInt(int64(port), 10), }, }) @@ -221,29 +232,48 @@ func (vllmLMCacheMPAdapter) InjectEngineConfig(pod *corev1.PodSpec, binding *bac return err } lm := cache.Spec.LMCache - if lm == nil || lm.Topology != cachev1alpha1.LMCacheTopologyPodLocal || lm.PodLocal == nil || lm.PodLocal.Server == nil { - return fmt.Errorf("inject vLLM LMCache MP: typed PodLocal server configuration is required") + if lm == nil { + return fmt.Errorf("inject vLLM LMCache MP: typed server configuration is required") } if !(vllmLMCacheMPAdapter{}).SupportsBinding(binding) { return fmt.Errorf("vLLM LMCache MP adapter does not support remote binding protocol %q", binding.Protocol) } - server := lm.PodLocal.Server - configJSON, err := vllmMPKVTransferConfigJSON(IntegrationRole(cache), server.Port) - if err != nil { - return err - } - work := pod.DeepCopy() - if _, err := renderLMCachePodLocalServer(work, EngineContainerName, lmCacheMPServerConfig{ - Image: server.Image, - Port: server.Port, - ChunkSizeTokens: effectiveLMCacheChunkSize(lm), - L1Capacity: server.L1Capacity, - MaxWorkers: server.MaxWorkers, - Resources: server.Resources, - Binding: binding, - WriteClientConfig: false, - }); err != nil { + var configJSON string + var err error + switch lm.Topology { + case cachev1alpha1.LMCacheTopologyPodLocal: + if lm.PodLocal == nil || lm.PodLocal.Server == nil { + return fmt.Errorf("inject vLLM LMCache MP: typed PodLocal server configuration is required") + } + server := lm.PodLocal.Server + configJSON, err = vllmMPKVTransferConfigJSON(IntegrationRole(cache), server.Port) + if err == nil { + _, err = renderLMCachePodLocalServer(work, EngineContainerName, lmCacheMPServerConfig{ + Image: server.Image, + Port: server.Port, + ChunkSizeTokens: effectiveLMCacheChunkSize(lm), + L1Capacity: server.L1Capacity, + MaxWorkers: server.MaxWorkers, + Resources: server.Resources, + Binding: binding, + WriteClientConfig: false, + }) + } + case cachev1alpha1.LMCacheTopologyNodeLocal: + if lm.NodeLocal == nil || lm.NodeLocal.Server == nil { + return fmt.Errorf("inject vLLM LMCache MP: typed NodeLocal server configuration is required") + } + server := lm.NodeLocal.Server + configJSON, err = vllmMPKVTransferConfigJSONForHost( + IntegrationRole(cache), "tcp://$("+lmCacheNodeIPEnv+")", server.Port) + if err == nil { + _, err = renderLMCacheNodeLocalEngine(work, EngineContainerName, cache, false) + } + default: + return fmt.Errorf("inject vLLM LMCache MP: topology %q is not implemented", lm.Topology) + } + if err != nil { return err } engineIndex, err := EngineContainerIndexNamed(work, EngineContainerName) diff --git a/internal/cli/doctor/checks/checks_test.go b/internal/cli/doctor/checks/checks_test.go index 309cc51e..ebb68741 100644 --- a/internal/cli/doctor/checks/checks_test.go +++ b/internal/cli/doctor/checks/checks_test.go @@ -790,6 +790,36 @@ func TestEnginePodInjectionAudit(t *testing.T) { t.Errorf("expected exactly 5 findings (one per matching pod), got %v", codesOf(fs)) } + t.Run("overlapping selectors report ambiguity without choosing a winner", func(t *testing.T) { + broad := backend.DeepCopy() + broad.Name = "alpha" + broad.UID = "alpha-uid" + narrow := backend.DeepCopy() + narrow.Name = "beta" + narrow.UID = "beta-uid" + narrow.Spec.EngineSelector.MatchLabels = map[string]string{"app": "engine", "model": "qwen"} + ambiguous := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "ambiguous", + Namespace: "ns1", + UID: "ambiguous-uid", + Labels: map[string]string{"app": "engine", "model": "qwen"}, + Annotations: map[string]string{ + annotationInjectedBy: "ns1/alpha", + annotationInjectedByUID: "alpha-uid", + }, + }} + fs := EnginePodInjectionAudit(ctx, fakeClient(t, broad, narrow, ambiguous), "") + if len(fs) != 1 { + t.Fatalf("want one ambiguity finding, got %+v", fs) + } + if fs[0].Code != doctor.CodeEngineSelectorAmbiguous || fs[0].Status != doctor.StatusWarn { + t.Fatalf("want EP003 WARN, got %+v", fs[0]) + } + if !strings.Contains(fs[0].Message, "alpha") || !strings.Contains(fs[0].Message, "beta") { + t.Fatalf("ambiguity message must name both backends: %q", fs[0].Message) + } + }) + t.Run("backend list error", func(t *testing.T) { c := listErrClient{Client: fakeClient(t), failOn: func(l client.ObjectList) bool { _, ok := l.(*cachev1alpha1.CacheBackendList) diff --git a/internal/cli/doctor/checks/podaudit.go b/internal/cli/doctor/checks/podaudit.go index 732137a7..d55059a7 100644 --- a/internal/cli/doctor/checks/podaudit.go +++ b/internal/cli/doctor/checks/podaudit.go @@ -23,12 +23,13 @@ const ( checkOrphanPods = "OrphanPodCheck" ) -// EnginePodInjectionAudit finds pods that match some CacheBackend's +// EnginePodInjectionAudit finds pods that match a CacheBackend's // engineSelector and verifies each carries an InjectedByCacheBackend Event — // the controller's proof that the mutating Pod webhook actually wired the pod. // A matched pod with no such Event is serving uncached (it likely lost the // admission race against the reconciler, or was created before the backend -// existed) and gets a WARN. +// existed) and gets a WARN. A pod matching multiple backends is also WARNed: +// there is no safe owner to audit, and fresh Pod admission would deny it. func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) []doctor.Finding { var backends cachev1alpha1.CacheBackendList if err := c.List(ctx, &backends, client.InNamespace(ns)); err != nil { @@ -51,10 +52,8 @@ func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) [] } byNamespace[cb.Namespace] = append(byNamespace[cb.Namespace], sel{backend: cb.Name, uid: string(cb.UID), labels: cb.Spec.EngineSelector.MatchLabels}) } - // Match the pod webhook's documented tie-break for overlapping selectors: - // the lexicographically-smallest CacheBackend name wins. Sorting here makes - // the audit's "first match" agree with the backend that actually injected - // the pod, rather than depending on List order. + // Sort for deterministic ambiguity messages. Unlike the historical status + // attribution fallback, doctor never chooses a winner for an invalid overlap. for ns := range byNamespace { sort.Slice(byNamespace[ns], func(i, j int) bool { return byNamespace[ns][i].backend < byNamespace[ns][j].backend }) } @@ -89,17 +88,31 @@ func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) [] var findings []doctor.Finding for i := range pods.Items { pod := &pods.Items[i] - var matched *sel + var matched []sel for j := range byNamespace[pod.Namespace] { if selectorMatches(byNamespace[pod.Namespace][j].labels, pod.Labels) { - matched = &byNamespace[pod.Namespace][j] - break + matched = append(matched, byNamespace[pod.Namespace][j]) } } - if matched == nil { + if len(matched) == 0 { continue } ref := resourceRef("Pod", pod.Namespace, pod.Name) + if len(matched) > 1 { + names := make([]string, 0, len(matched)) + for _, candidate := range matched { + names = append(names, candidate.backend) + } + findings = append(findings, doctor.Finding{ + Code: doctor.CodeEngineSelectorAmbiguous, + Status: doctor.StatusWarn, + Check: checkEnginePodInjection, + Resource: ref, + Message: fmt.Sprintf("engine pod matches multiple CacheBackends %q in namespace %q; no backend is selected implicitly, and fresh Pod admission would deny this ambiguity — make the engineSelectors disjoint", names, pod.Namespace), + }) + continue + } + selected := matched[0] // Trust the durable inferencecache.io/injected-by annotation only when it // both NAMES the matched backend and carries an injected-by-uid matching @@ -107,8 +120,8 @@ func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) [] // which rejects a forged, stale, or internally-inconsistent annotation // pair. This outlives the GC-able Event. if owner := pod.Annotations[annotationInjectedBy]; owner != "" && - owner == pod.Namespace+"/"+matched.backend && - matched.uid != "" && pod.Annotations[annotationInjectedByUID] == matched.uid { + owner == pod.Namespace+"/"+selected.backend && + selected.uid != "" && pod.Annotations[annotationInjectedByUID] == selected.uid { findings = append(findings, doctor.Finding{ Code: doctor.CodeEnginePodInjected, Status: doctor.StatusOK, @@ -135,7 +148,7 @@ func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) [] Status: doctor.StatusOK, Check: checkEnginePodInjection, Resource: ref, - Message: fmt.Sprintf("engine pod matches CacheBackend %q (engineSelector) and carries an InjectedByCacheBackend Event for its current UID — it was injected by the cache plane (the Event does not record which backend; the inferencecache.io/injected-by annotation is the authoritative per-backend signal and is absent or unvalidated here)", matched.backend), + Message: fmt.Sprintf("engine pod matches CacheBackend %q (engineSelector) and carries an InjectedByCacheBackend Event for its current UID — it was injected by the cache plane (the Event does not record which backend; the inferencecache.io/injected-by annotation is the authoritative per-backend signal and is absent or unvalidated here)", selected.backend), }) } else { findings = append(findings, doctor.Finding{ @@ -143,7 +156,7 @@ func EnginePodInjectionAudit(ctx context.Context, c client.Client, ns string) [] Status: doctor.StatusWarn, Check: checkEnginePodInjection, Resource: ref, - Message: fmt.Sprintf("engine pod matches CacheBackend %q (engineSelector) but has no injection marker (no validated inferencecache.io/injected-by annotation and no InjectedByCacheBackend Event for its UID) — it may be running uncached; recreate it (e.g. kubectl rollout restart) so the mutating webhook re-evaluates", matched.backend), + Message: fmt.Sprintf("engine pod matches CacheBackend %q (engineSelector) but has no injection marker (no validated inferencecache.io/injected-by annotation and no InjectedByCacheBackend Event for its UID) — it may be running uncached; recreate it (e.g. kubectl rollout restart) so the mutating webhook re-evaluates", selected.backend), }) } } diff --git a/internal/cli/doctor/finding.go b/internal/cli/doctor/finding.go index 5acc4ca3..b8a86adc 100644 --- a/internal/cli/doctor/finding.go +++ b/internal/cli/doctor/finding.go @@ -173,6 +173,11 @@ const ( // the validated inferencecache.io/injected-by annotation (authoritative, // names the backend) or an InjectedByCacheBackend Event for its UID. CodeEnginePodInjected = "EP002" + // CodeEngineSelectorAmbiguous: a pod is selected by more than one + // same-namespace CacheBackend. Admission rejects this configuration for new + // objects; seeing it indicates admission-bypassed or concurrently-created state + // that has no safe implicit owner. + CodeEngineSelectorAmbiguous = "EP003" // CodeOrphanPod: a pod has a NoMatchingCacheBackend Event — it expected // injection but matched no CacheBackend (likely operator misconfiguration). diff --git a/internal/controller/cachebackend_dispatch.go b/internal/controller/cachebackend_dispatch.go index 1f7a2a1e..db7c19a0 100644 --- a/internal/controller/cachebackend_dispatch.go +++ b/internal/controller/cachebackend_dispatch.go @@ -27,6 +27,11 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge registry := r.Registry runtimeID := adapterruntime.ResolveRuntimeID(backend) storage := backend.Spec.EffectiveRemoteStorage() + if !isTypedLMCacheNodeLocal(backend) { + if err := r.cleanupLMCacheNodeLocalServerPods(ctx, backend); err != nil { + return ctrl.Result{}, err + } + } // Events-only (tier-1 routing) provisions no backend server: the engine is // wired for cache-aware routing via the kvevent-subscriber alone, with no KV @@ -102,6 +107,9 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge if err := r.cleanupOwnedWorkload(ctx, backend); err != nil { return ctrl.Result{}, err } + if err := r.reconcileLMCacheNodeLocalServerPods(ctx, backend, binding); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{}, r.reconcileExternal(ctx, backend) } @@ -120,6 +128,9 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge if backend.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeSGLangHiCache { return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) } + if err := r.reconcileLMCacheNodeLocalServerPods(ctx, backend, nil); err != nil { + return ctrl.Result{}, err + } return r.reconcileHostOnly(ctx, backend) } @@ -134,13 +145,17 @@ func (r *CacheBackendReconciler) dispatch(ctx context.Context, logger logr.Logge if err != nil { return ctrl.Result{}, fmt.Errorf("render remote storage for %s/%s: %w", backend.Namespace, backend.Name, err) } - binding := &backendadapter.Binding{Protocol: rendered.Protocol} + desiredService := r.buildService(backend, rendered.Service) + binding := backendadapter.BindingFor(storage, rendered.Protocol, serviceEndpoint(desiredService)) if !adapter.SupportsBinding(binding) { logger.V(1).Info("runtime adapter does not accept remote-storage binding; treating as unmanaged", "runtime", runtimeID, "type", backend.Spec.EffectiveCacheType(), "protocol", rendered.Protocol, "namespace", backend.Namespace, "name", backend.Name) return ctrl.Result{}, r.reconcileUnmanaged(ctx, backend) } + if err := r.reconcileLMCacheNodeLocalServerPods(ctx, backend, binding); err != nil { + return ctrl.Result{}, err + } return r.reconcileManaged(ctx, logger, backend, rendered) } diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index be647532..a7c80bf0 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -7,7 +7,9 @@ package controller import ( "context" "fmt" + "sort" "strconv" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -18,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" "github.com/cachebox-project/inference-cache/internal/enginebinding" ) @@ -25,14 +28,18 @@ const ( conditionTypeConnectorReady = "ConnectorReady" conditionTypeRemoteStorageReady = "RemoteStorageReady" - reasonConnectorReady = "ConnectorReady" - reasonConnectorUnverified = "ConnectorInjectionUnverified" - reasonNoEnginePods = "NoEnginePods" - reasonMPServersNotReady = "MPServersNotReady" - reasonRemoteStorageReady = "RemoteStorageReady" - reasonRemoteStorageAbsent = "RemoteStorageNotConfigured" - reasonRemoteStoragePending = "RemoteStoragePending" - reasonRemoteStorageUnavailable = "RemoteStorageUnavailable" + reasonConnectorReady = "ConnectorReady" + reasonConnectorUnverified = "ConnectorInjectionUnverified" + reasonNoEnginePods = "NoEnginePods" + reasonMPServersNotReady = "MPServersNotReady" + reasonNodeLocalPoolPending = "NodeLocalServerPoolPending" + reasonNodeLocalHostPortConflict = "NodeLocalHostPortConflict" + reasonNodeLocalWorkerCapacity = "NodeLocalWorkerCapacityExceeded" + reasonNodeLocalAmbiguousServers = "AmbiguousNodeLocalServers" + reasonRemoteStorageReady = "RemoteStorageReady" + reasonRemoteStorageAbsent = "RemoteStorageNotConfigured" + reasonRemoteStoragePending = "RemoteStoragePending" + reasonRemoteStorageUnavailable = "RemoteStorageUnavailable" lmCacheMPServerStatusContainerName = "lmcache-mp-server" ) @@ -52,7 +59,7 @@ func isTypedLMCachePodLocal(backend *cachev1alpha1.CacheBackend) bool { // cluster-wide Pod watch. List/patch errors preserve the prior verdict and are // fail-soft so connector observability cannot block normal reconciliation. func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend) { - if !isTypedLMCachePodLocal(backend) { + if !isTypedLMCacheMP(backend) { if backend.Status.Connector == nil && meta.FindStatusCondition(backend.Status.Conditions, conditionTypeConnectorReady) == nil { return } @@ -66,6 +73,10 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con } return } + if isTypedLMCacheNodeLocal(backend) { + r.refreshLMCacheNodeLocalConnectorStatus(ctx, backend) + return + } reader := client.Reader(r.APIReader) if reader == nil { @@ -85,6 +96,7 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con } var matched, verified, readyEngines, desiredServers, readyServers, covered int32 + engineCoverage := make([]cachev1alpha1.CacheBackendEnginePodCoverageStatus, 0, len(pods.Items)) wantInjectedBy := backend.Namespace + "/" + backend.Name wantUID := string(backend.UID) for i := range pods.Items { @@ -102,6 +114,15 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con verified++ } serverReady := injected && nativeSidecarReady(pod.Status.InitContainerStatuses, lmCacheMPServerStatusContainerName) + coverageReason := reasonMPServersNotReady + if !injected { + coverageReason = reasonConnectorUnverified + } else if serverReady { + coverageReason = reasonConnectorReady + } + engineCoverage = append(engineCoverage, cachev1alpha1.CacheBackendEnginePodCoverageStatus{ + Name: pod.Name, NodeName: pod.Spec.NodeName, Ready: podReady(pod), Covered: serverReady, Reason: coverageReason, + }) if serverReady { readyServers++ covered++ @@ -110,6 +131,7 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con } } } + sort.Slice(engineCoverage, func(i, j int) bool { return engineCoverage[i].Name < engineCoverage[j].Name }) connector := &cachev1alpha1.CacheBackendConnectorStatus{ Mode: cachev1alpha1.LMCacheConnectorModeMultiprocess, @@ -120,6 +142,7 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con ReadyServers: readyServers, CoveredEnginePods: covered, UncoveredEnginePods: matched - covered, + EnginePodCoverage: engineCoverage, } status, reason, message := metav1.ConditionFalse, reasonMPServersNotReady, fmt.Sprintf("%d/%d selected engine Pods have a Ready LMCache MP native sidecar; %d engine Pods are Ready with the connector", readyServers, desiredServers, readyEngines) @@ -155,6 +178,225 @@ func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Con } } +func (r *CacheBackendReconciler) refreshLMCacheNodeLocalConnectorStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend) { + // Admission rejects incomplete NodeLocal objects, but a controller can still + // encounter an older or admission-bypassed object. Dispatch reports the + // renderer error; keep status refresh fail-soft instead of dereferencing a + // missing server declaration on that error path. + if backend == nil || backend.Spec.LMCache == nil || backend.Spec.LMCache.NodeLocal == nil || + backend.Spec.LMCache.NodeLocal.Server == nil { + log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: server configuration is incomplete") + return + } + reader := client.Reader(r.APIReader) + if reader == nil { + reader = r.Client + } + var engines corev1.PodList + selector := backend.Spec.EngineSelector + if selector != nil && len(selector.MatchLabels) > 0 { + if err := reader.List(ctx, &engines, + client.InNamespace(backend.Namespace), + client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(selector.MatchLabels)}, + ); err != nil { + log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: engine pod list failed", "error", err.Error()) + return + } + } + + var servers corev1.PodList + if err := reader.List(ctx, &servers, + client.InNamespace(backend.Namespace), + client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }, + ); err != nil { + log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: server pod list failed", "error", err.Error()) + return + } + + wantOwner := backend.Namespace + "/" + backend.Name + wantUID := string(backend.UID) + wantGeneration := strconv.FormatInt(backend.Generation, 10) + wantShmName, shmNameErr := builtinruntime.NodeLocalServerShmName(backend) + if shmNameErr != nil { + log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: shared-memory identity is invalid", "error", shmNameErr.Error()) + return + } + readyByNode := map[string]int32{} + conflictByNode := map[string]bool{} + for i := range servers.Items { + pod := &servers.Items[i] + if !metav1.IsControlledBy(pod, backend) || pod.DeletionTimestamp != nil { + continue + } + annotations := pod.GetAnnotations() + if annotations[enginebinding.AnnotationNodeLocalOwner] != wantOwner || + annotations[enginebinding.AnnotationNodeLocalOwnerUID] != wantUID || + annotations[enginebinding.AnnotationNodeLocalGeneration] != wantGeneration || + !nodeLocalServerHasShmIdentity(pod, wantShmName) { + continue + } + targetNode := annotations[enginebinding.AnnotationNodeLocalTargetNode] + if nodeLocalHostPortConflict(pod) { + conflictByNode[targetNode] = true + } + if targetNode != "" && pod.Spec.NodeName == targetNode && podReady(pod) && normalContainerReady(pod.Status.ContainerStatuses, lmCacheMPServerStatusContainerName) { + readyByNode[targetNode]++ + } + } + activeEngines := make([]*corev1.Pod, 0, len(engines.Items)) + engineCountByNode := map[string]int32{} + desiredNodes := map[string]struct{}{} + for i := range engines.Items { + pod := &engines.Items[i] + if pod.DeletionTimestamp != nil || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + activeEngines = append(activeEngines, pod) + annotations := pod.GetAnnotations() + owned := annotations[enginebinding.AnnotationInjectedBy] == wantOwner && + wantUID != "" && annotations[enginebinding.AnnotationInjectedByUID] == wantUID + if owned && pod.Spec.NodeName != "" { + engineCountByNode[pod.Spec.NodeName]++ + desiredNodes[pod.Spec.NodeName] = struct{}{} + } + } + desiredServers := int32(len(desiredNodes)) + readyServers := int32(0) + hostPortConflict := false + for node := range desiredNodes { + if readyByNode[node] == 1 { + readyServers++ + } + if conflictByNode[node] { + hostPortConflict = true + } + } + matched := int32(len(activeEngines)) + verified, covered, readyEngines := int32(0), int32(0), int32(0) + engineCoverage := make([]cachev1alpha1.CacheBackendEnginePodCoverageStatus, 0, len(activeEngines)) + workerCapacityExceeded := false + ambiguousServers := false + maxGPUWorkers := backend.Spec.LMCache.NodeLocal.Server.MaxGPUWorkers + for _, pod := range activeEngines { + annotations := pod.GetAnnotations() + injected := annotations[enginebinding.AnnotationInjectedBy] == wantOwner && + wantUID != "" && annotations[enginebinding.AnnotationInjectedByUID] == wantUID && + annotations[enginebinding.AnnotationInjectedGeneration] == wantGeneration + if injected { + verified++ + } + node := pod.Spec.NodeName + withinWorkers := node != "" && engineCountByNode[node] > 0 && engineCountByNode[node] <= maxGPUWorkers + if injected && node != "" && !withinWorkers { + workerCapacityExceeded = true + } + isCovered := injected && withinWorkers && readyByNode[node] == 1 + coverageReason := reasonMPServersNotReady + switch { + case !injected: + coverageReason = reasonConnectorUnverified + case node == "": + coverageReason = "EngineSchedulingPending" + case !withinWorkers: + coverageReason = reasonNodeLocalWorkerCapacity + case readyByNode[node] > 1: + ambiguousServers = true + coverageReason = reasonNodeLocalAmbiguousServers + case isCovered: + coverageReason = reasonConnectorReady + } + engineCoverage = append(engineCoverage, cachev1alpha1.CacheBackendEnginePodCoverageStatus{ + Name: pod.Name, NodeName: node, Ready: podReady(pod), Covered: isCovered, Reason: coverageReason, + }) + if isCovered { + covered++ + if podReady(pod) { + readyEngines++ + } + } + } + sort.Slice(engineCoverage, func(i, j int) bool { return engineCoverage[i].Name < engineCoverage[j].Name }) + + connector := &cachev1alpha1.CacheBackendConnectorStatus{ + Mode: cachev1alpha1.LMCacheConnectorModeMultiprocess, + Topology: cachev1alpha1.LMCacheTopologyNodeLocal, + MatchedEnginePods: matched, + ReadyEnginePods: readyEngines, + DesiredServers: desiredServers, + ReadyServers: readyServers, + CoveredEnginePods: covered, + UncoveredEnginePods: matched - covered, + EnginePodCoverage: engineCoverage, + } + status, reason, message := metav1.ConditionFalse, reasonNodeLocalPoolPending, + fmt.Sprintf("%d/%d engine-demanded LMCache MP servers are Ready; %d/%d selected engine Pods have exactly one healthy same-node server", readyServers, desiredServers, covered, matched) + switch { + case matched == 0: + reason = reasonNoEnginePods + message = "no active engine Pods match spec.engineSelector" + case verified != matched: + status = metav1.ConditionUnknown + reason = reasonConnectorUnverified + message = fmt.Sprintf("%d/%d selected engine Pods carry the current CacheBackend name, UID, and generation injection record", verified, matched) + case desiredServers == 0: + reason = reasonNodeLocalPoolPending + message = "selected engine Pods have not been scheduled onto a node yet" + case hostPortConflict: + reason = reasonNodeLocalHostPortConflict + message = "a NodeLocal server Pod is unschedulable because a declared host port is already allocated; choose disjoint MP and HTTP ports" + case workerCapacityExceeded: + reason = reasonNodeLocalWorkerCapacity + message = fmt.Sprintf("at least one node has more selected engine instances than maxGPUWorkers=%d", maxGPUWorkers) + case ambiguousServers: + reason = reasonNodeLocalAmbiguousServers + message = "more than one healthy current-generation NodeLocal server claims the same engine node" + case readyServers == desiredServers && covered == matched && readyEngines == matched: + status = metav1.ConditionTrue + reason = reasonConnectorReady + message = fmt.Sprintf("all %d selected engine Pods are Ready and covered by exactly one healthy same-node server; all %d engine-demanded servers are Ready", matched, desiredServers) + } + + before := backend.DeepCopy() + backend.Status.Connector = connector + meta.SetStatusCondition(&backend.Status.Conditions, metav1.Condition{ + Type: conditionTypeConnectorReady, Status: status, Reason: reason, Message: message, + ObservedGeneration: backend.Generation, + }) + if equality.Semantic.DeepEqual(before.Status, backend.Status) { + return + } + if err := r.Status().Patch(ctx, backend, client.MergeFrom(before)); err != nil { + backend.Status = before.Status + log.FromContext(ctx).V(1).Info("LMCache NodeLocal connector status refresh skipped: patch failed", "error", err.Error()) + } +} + +func normalContainerReady(statuses []corev1.ContainerStatus, name string) bool { + for i := range statuses { + if statuses[i].Name == name { + return statuses[i].Ready && statuses[i].State.Running != nil + } + } + return false +} + +func nodeLocalHostPortConflict(pod *corev1.Pod) bool { + for i := range pod.Status.Conditions { + condition := pod.Status.Conditions[i] + if condition.Type != corev1.PodScheduled || condition.Status != corev1.ConditionFalse || condition.Reason != corev1.PodReasonUnschedulable { + continue + } + message := strings.ToLower(condition.Message) + if strings.Contains(message, "free ports") || strings.Contains(message, "hostport") || strings.Contains(message, "host port") { + return true + } + } + return false +} + func nativeSidecarReady(statuses []corev1.ContainerStatus, name string) bool { for i := range statuses { if statuses[i].Name == name { @@ -173,7 +415,7 @@ func podReady(pod *corev1.Pod) bool { return false } -// lmCacheMPReadyBase aggregates the required PodLocal connector and optional +// lmCacheMPReadyBase aggregates the required typed MP connector and optional // remote L3 without hiding either component's dedicated condition. The // connector always gates Ready because SGLang cannot run with // --enable-lmcache when its co-scheduled MP server is unavailable. Remote @@ -189,7 +431,7 @@ func lmCacheMPReadyBase( remoteStatus metav1.ConditionStatus, remoteReason, remoteMessage string, ) (metav1.ConditionStatus, string, string) { - if !isTypedLMCachePodLocal(backend) { + if !isTypedLMCacheMP(backend) { return remoteStatus, remoteReason, remoteMessage } @@ -208,13 +450,13 @@ func lmCacheMPReadyBase( } if storage != nil && remoteStatus != metav1.ConditionTrue { return metav1.ConditionTrue, reasonConnectorReady, - "the Pod-local MP connector is ready; remote storage is degraded but does not gate Ready while failOpen is true" + "the LMCache MP connector is ready; remote storage is degraded but does not gate Ready while failOpen is true" } return metav1.ConditionTrue, reasonConnectorReady, connector.Message } func setRemoteStorageStatus(backend *cachev1alpha1.CacheBackend, endpoint string, ready metav1.ConditionStatus, reason, message string, observedGeneration int64) { - if !isTypedLMCachePodLocal(backend) { + if !isTypedLMCacheMP(backend) { backend.Status.RemoteStorage = nil meta.RemoveStatusCondition(&backend.Status.Conditions, conditionTypeRemoteStorageReady) return diff --git a/internal/controller/cachebackend_lmcache_mp_status_test.go b/internal/controller/cachebackend_lmcache_mp_status_test.go index 12e642aa..340ba012 100644 --- a/internal/controller/cachebackend_lmcache_mp_status_test.go +++ b/internal/controller/cachebackend_lmcache_mp_status_test.go @@ -15,12 +15,27 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" "github.com/cachebox-project/inference-cache/internal/enginebinding" ) +func setNodeLocalShmIdentity(t *testing.T, backend *cachev1alpha1.CacheBackend, pod *corev1.Pod) { + t.Helper() + name, err := builtinruntime.NodeLocalServerShmName(backend) + if err != nil { + t.Fatal(err) + } + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[enginebinding.AnnotationNodeLocalShmName] = name + pod.Spec.Containers = []corev1.Container{{Name: lmCacheMPServerStatusContainerName, Args: []string{"server", "--shm-name", name}}} +} + func typedMPStatusBackend() *cachev1alpha1.CacheBackend { return &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "cache", Namespace: "ns1", UID: types.UID("cache-uid"), Generation: 3}, @@ -89,6 +104,10 @@ func TestRefreshLMCacheMPConnectorStatusTransitions(t *testing.T) { status.ReadyEnginePods != 1 || status.CoveredEnginePods != 1 || status.UncoveredEnginePods != 1 { t.Fatalf("connector status = %+v", status) } + if len(status.EnginePodCoverage) != 2 || status.EnginePodCoverage[0].Name != "ready" || !status.EnginePodCoverage[0].Covered || + status.EnginePodCoverage[1].Name != "uncovered" || status.EnginePodCoverage[1].Covered { + t.Fatalf("engine pod coverage = %+v", status.EnginePodCoverage) + } cond := meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) if cond == nil || cond.Status != metav1.ConditionUnknown || cond.Reason != reasonConnectorUnverified { t.Fatalf("ConnectorReady = %+v", cond) @@ -153,6 +172,245 @@ func TestRefreshLMCacheMPConnectorStatusTransitions(t *testing.T) { } } +func TestRefreshLMCacheNodeLocalConnectorStatusSameNodeCoverage(t *testing.T) { + ctx := context.Background() + scheme := newScheme(t) + backend := nodeLocalBackend("cache", "ns1") + backend.Generation = 3 + backend.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "sglang"}} + controller := true + serverPod := func(name, node string, ready bool) *corev1.Pod { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "ns1", + Labels: map[string]string{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }, + Annotations: map[string]string{ + enginebinding.AnnotationNodeLocalOwner: "ns1/cache", + enginebinding.AnnotationNodeLocalOwnerUID: string(backend.UID), + enginebinding.AnnotationNodeLocalGeneration: "3", + enginebinding.AnnotationNodeLocalTargetNode: node, + }, + OwnerReferences: []metav1.OwnerReference{{APIVersion: cachev1alpha1.GroupVersion.String(), Kind: "CacheBackend", Name: backend.Name, UID: backend.UID, Controller: &controller}}, + }, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: lmCacheMPServerStatusContainerName, Ready: ready, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } + if ready { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + setNodeLocalShmIdentity(t, backend, pod) + return pod + } + enginePod := func(name, node string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "ns1", Labels: map[string]string{"app": "sglang"}, + Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: "ns1/cache", + enginebinding.AnnotationInjectedByUID: string(backend.UID), + enginebinding.AnnotationInjectedGeneration: "3", + }, + }, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, + } + } + objects := []client.Object{ + backend, + serverPod("server-a", "node-a", true), serverPod("server-b", "node-b", true), + enginePod("engine-a1", "node-a"), enginePod("engine-a2", "node-a"), enginePod("engine-b1", "node-b"), + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &corev1.Pod{}). + WithObjects(objects...).Build() + r := &CacheBackendReconciler{Client: c, APIReader: c} + r.refreshLMCacheMPConnectorStatus(ctx, backend) + + var got cachev1alpha1.CacheBackend + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get backend: %v", err) + } + if got.Status.Connector == nil || got.Status.Connector.DesiredServers != 2 || got.Status.Connector.ReadyServers != 2 || + got.Status.Connector.MatchedEnginePods != 3 || got.Status.Connector.CoveredEnginePods != 3 || got.Status.Connector.ReadyEnginePods != 3 { + t.Fatalf("NodeLocal connector status = %+v", got.Status.Connector) + } + if len(got.Status.Connector.EnginePodCoverage) != 3 || got.Status.Connector.EnginePodCoverage[0].Name != "engine-a1" || + !got.Status.Connector.EnginePodCoverage[0].Covered || got.Status.Connector.EnginePodCoverage[2].NodeName != "node-b" { + t.Fatalf("NodeLocal engine pod coverage = %+v", got.Status.Connector.EnginePodCoverage) + } + condition := meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) + if condition == nil || condition.Status != metav1.ConditionTrue { + t.Fatalf("ConnectorReady = %+v", condition) + } + + // Losing only node-a's server uncovers both node-a engines while the + // node-b engine remains covered; coverage never falls back to another node. + var serverA corev1.Pod + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "server-a"}, &serverA); err != nil { + t.Fatalf("get server-a: %v", err) + } + serverA.Status.ContainerStatuses[0].Ready = false + serverA.Status.Conditions = nil + if err := c.Status().Update(ctx, &serverA); err != nil { + t.Fatalf("mark server-a unready: %v", err) + } + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, backend); err != nil { + t.Fatalf("refresh backend: %v", err) + } + r.refreshLMCacheMPConnectorStatus(ctx, backend) + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get degraded backend: %v", err) + } + if got.Status.Connector.CoveredEnginePods != 1 || got.Status.Connector.UncoveredEnginePods != 2 || got.Status.Connector.ReadyServers != 1 { + t.Fatalf("same-node degraded coverage = %+v", got.Status.Connector) + } +} + +func TestNodeLocalHostPortConflictReason(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: corev1.PodReasonUnschedulable, + Message: "0/2 nodes are available: 2 node(s) didn't have free ports for the requested pod ports", + }}}} + if !nodeLocalHostPortConflict(pod) { + t.Fatal("scheduler host-port conflict was not classified") + } +} + +func TestRefreshLMCacheNodeLocalConnectorStatusFailureModes(t *testing.T) { + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend, *[]*corev1.Pod, *corev1.Pod) + wantCondition string + wantCoverage string + wantReadyServers int32 + noEngine bool + }{ + { + name: "no selected engines", mutate: func(_ *cachev1alpha1.CacheBackend, _ *[]*corev1.Pod, _ *corev1.Pod) {}, + wantCondition: reasonNoEnginePods, wantReadyServers: 0, noEngine: true, + }, + { + name: "engine has not been admitted with current identity", + mutate: func(_ *cachev1alpha1.CacheBackend, _ *[]*corev1.Pod, engine *corev1.Pod) { + engine.Annotations[enginebinding.AnnotationInjectedGeneration] = "2" + }, + wantCondition: reasonConnectorUnverified, wantCoverage: reasonConnectorUnverified, wantReadyServers: 1, + }, + { + name: "engine is not scheduled", + mutate: func(_ *cachev1alpha1.CacheBackend, _ *[]*corev1.Pod, engine *corev1.Pod) { + engine.Spec.NodeName = "" + }, + wantCondition: reasonNodeLocalPoolPending, wantCoverage: "EngineSchedulingPending", wantReadyServers: 0, + }, + { + name: "worker capacity exceeded", + mutate: func(backend *cachev1alpha1.CacheBackend, _ *[]*corev1.Pod, _ *corev1.Pod) { + backend.Spec.LMCache.NodeLocal.Server.MaxGPUWorkers = 0 + }, + wantCondition: reasonNodeLocalWorkerCapacity, wantCoverage: reasonNodeLocalWorkerCapacity, wantReadyServers: 1, + }, + { + name: "server is missing UID-scoped shared-memory identity", + mutate: func(_ *cachev1alpha1.CacheBackend, servers *[]*corev1.Pod, _ *corev1.Pod) { + delete((*servers)[0].Annotations, enginebinding.AnnotationNodeLocalShmName) + }, + wantCondition: reasonNodeLocalPoolPending, wantCoverage: reasonMPServersNotReady, wantReadyServers: 0, + }, + { + name: "ambiguous ready servers on one node", + mutate: func(_ *cachev1alpha1.CacheBackend, servers *[]*corev1.Pod, _ *corev1.Pod) { + duplicate := (*servers)[0].DeepCopy() + duplicate.Name = "server-duplicate" + *servers = append(*servers, duplicate) + }, + wantCondition: reasonNodeLocalAmbiguousServers, wantCoverage: reasonNodeLocalAmbiguousServers, wantReadyServers: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + backend := nodeLocalBackend("cache", "ns1") + backend.Generation = 3 + backend.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}} + controller := true + server := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "server", Namespace: "ns1", + Labels: map[string]string{enginebinding.LabelLMCacheNodeLocalServer: "true", enginebinding.LabelCacheBackendUID: string(backend.UID)}, + Annotations: map[string]string{ + enginebinding.AnnotationNodeLocalOwner: "ns1/cache", enginebinding.AnnotationNodeLocalOwnerUID: string(backend.UID), + enginebinding.AnnotationNodeLocalGeneration: "3", enginebinding.AnnotationNodeLocalTargetNode: "node-a", + }, + OwnerReferences: []metav1.OwnerReference{{APIVersion: cachev1alpha1.GroupVersion.String(), Kind: "CacheBackend", Name: backend.Name, UID: backend.UID, Controller: &controller}}, + }, + Spec: corev1.PodSpec{NodeName: "node-a"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + ContainerStatuses: []corev1.ContainerStatus{{Name: lmCacheMPServerStatusContainerName, Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}, + }, + } + setNodeLocalShmIdentity(t, backend, server) + servers := []*corev1.Pod{server} + engine := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "engine", Namespace: "ns1", Labels: map[string]string{"app": "engine"}, Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: "ns1/cache", enginebinding.AnnotationInjectedByUID: string(backend.UID), enginebinding.AnnotationInjectedGeneration: "3", + }}, + Spec: corev1.PodSpec{NodeName: "node-a"}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, + } + tt.mutate(backend, &servers, engine) + objects := []client.Object{backend} + if !tt.noEngine { + objects = append(objects, engine) + } + for _, server := range servers { + objects = append(objects, server) + } + c := fake.NewClientBuilder().WithScheme(newScheme(t)).WithStatusSubresource(&cachev1alpha1.CacheBackend{}, &corev1.Pod{}).WithObjects(objects...).Build() + r := &CacheBackendReconciler{Client: c, APIReader: c} + r.refreshLMCacheNodeLocalConnectorStatus(ctx, backend) + + var got cachev1alpha1.CacheBackend + if err := c.Get(ctx, types.NamespacedName{Namespace: "ns1", Name: "cache"}, &got); err != nil { + t.Fatalf("get backend: %v", err) + } + condition := meta.FindStatusCondition(got.Status.Conditions, conditionTypeConnectorReady) + if condition == nil || condition.Reason != tt.wantCondition { + t.Fatalf("ConnectorReady = %+v, want reason %s", condition, tt.wantCondition) + } + if got.Status.Connector == nil || got.Status.Connector.ReadyServers != tt.wantReadyServers { + t.Fatalf("connector status = %+v, want coverage reason %s and %d ready servers", got.Status.Connector, tt.wantCoverage, tt.wantReadyServers) + } + if tt.noEngine { + if len(got.Status.Connector.EnginePodCoverage) != 0 { + t.Fatalf("unexpected engine coverage = %+v", got.Status.Connector.EnginePodCoverage) + } + } else if len(got.Status.Connector.EnginePodCoverage) != 1 || got.Status.Connector.EnginePodCoverage[0].Reason != tt.wantCoverage { + t.Fatalf("engine coverage = %+v, want reason %s", got.Status.Connector.EnginePodCoverage, tt.wantCoverage) + } + }) + } +} + +func TestRefreshLMCacheNodeLocalConnectorStatusIncompleteObjectIsFailSoft(t *testing.T) { + backend := nodeLocalBackend("cache", "ns1") + backend.Spec.LMCache.NodeLocal.Server = nil + r := &CacheBackendReconciler{} + r.refreshLMCacheNodeLocalConnectorStatus(context.Background(), backend) + if backend.Status.Connector != nil { + t.Fatalf("incomplete admission-bypassed object unexpectedly received connector status: %+v", backend.Status.Connector) + } +} + func TestSetRemoteStorageStatusIndependentFromConnector(t *testing.T) { backend := typedMPStatusBackend() backend.Spec.RemoteStorage = &cachev1alpha1.CacheBackendRemoteStorageSpec{ diff --git a/internal/controller/cachebackend_lmcache_nodelocal.go b/internal/controller/cachebackend_lmcache_nodelocal.go new file mode 100644 index 00000000..aab6ddb3 --- /dev/null +++ b/internal/controller/cachebackend_lmcache_nodelocal.go @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" + "github.com/cachebox-project/inference-cache/internal/enginebinding" + backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" +) + +func isTypedLMCacheNodeLocal(backend *cachev1alpha1.CacheBackend) bool { + return backend != nil && + backend.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && + backend.Spec.LMCache != nil && + backend.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyNodeLocal +} + +func isTypedLMCacheMP(backend *cachev1alpha1.CacheBackend) bool { + return isTypedLMCachePodLocal(backend) || isTypedLMCacheNodeLocal(backend) +} + +// reconcileLMCacheNodeLocalServerPods follows the inference system's actual +// placement. One server Pod is created for every distinct node that currently +// hosts an active engine injected for this CacheBackend. Engines remain the +// scheduling authority; an unscheduled engine never causes speculative server +// placement. +func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context.Context, backend *cachev1alpha1.CacheBackend, binding *backendadapter.Binding) error { + if !isTypedLMCacheNodeLocal(backend) { + return r.cleanupLMCacheNodeLocalServerPods(ctx, backend) + } + demand, err := r.nodeLocalEngineDemand(ctx, backend) + if err != nil { + return err + } + wantShmName, err := builtinruntime.NodeLocalServerShmName(backend) + if err != nil { + return err + } + + var servers corev1.PodList + if err := r.Client.List(ctx, &servers, + client.InNamespace(backend.Namespace), + client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }, + ); err != nil { + return fmt.Errorf("list LMCache NodeLocal server Pods for %s/%s: %w", backend.Namespace, backend.Name, err) + } + + wantGeneration := strconv.FormatInt(backend.Generation, 10) + liveByNode := make(map[string]*corev1.Pod, len(servers.Items)) + for i := range servers.Items { + pod := &servers.Items[i] + if !metav1.IsControlledBy(pod, backend) { + continue + } + targetNode := pod.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + _, wanted := demand[targetNode] + current := pod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID] == string(backend.UID) && + pod.Annotations[enginebinding.AnnotationNodeLocalGeneration] == wantGeneration && + pod.Name == builtinruntime.NodeLocalServerPodName(backend.Name, targetNode) && + nodeLocalServerHasShmIdentity(pod, wantShmName) + if !current { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete stale LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + if !wanted { + retention := time.Duration(backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds) * time.Second + if retention <= 0 { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete idle LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + idleSince, parseErr := time.Parse(time.RFC3339Nano, pod.Annotations[enginebinding.AnnotationNodeLocalIdleSince]) + if parseErr != nil { + before := pod.DeepCopy() + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[enginebinding.AnnotationNodeLocalIdleSince] = time.Now().UTC().Format(time.RFC3339Nano) + if err := r.Client.Patch(ctx, pod, client.MergeFrom(before)); err != nil { + return fmt.Errorf("mark LMCache NodeLocal server Pod %s/%s idle: %w", pod.Namespace, pod.Name, err) + } + continue + } + if time.Since(idleSince) >= retention && pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete expired idle LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + if _, wasIdle := pod.Annotations[enginebinding.AnnotationNodeLocalIdleSince]; wasIdle { + before := pod.DeepCopy() + delete(pod.Annotations, enginebinding.AnnotationNodeLocalIdleSince) + if err := r.Client.Patch(ctx, pod, client.MergeFrom(before)); err != nil { + return fmt.Errorf("reactivate LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + if prior := liveByNode[targetNode]; prior != nil { + if pod.DeletionTimestamp == nil { + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete duplicate LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + continue + } + liveByNode[targetNode] = pod + } + + nodes := make([]string, 0, len(demand)) + for nodeName := range demand { + nodes = append(nodes, nodeName) + } + sort.Strings(nodes) + for _, nodeName := range nodes { + if liveByNode[nodeName] != nil { + continue + } + desired, err := builtinruntime.RenderLMCacheNodeLocalServerPod(backend, binding, nodeName, demand[nodeName]) + if err != nil { + return err + } + if err := controllerutil.SetControllerReference(backend, desired, r.Scheme); err != nil { + return fmt.Errorf("own LMCache NodeLocal server Pod %s/%s: %w", desired.Namespace, desired.Name, err) + } + if err := r.Client.Create(ctx, desired); err != nil { + if apierrors.IsAlreadyExists(err) { + var existing corev1.Pod + if getErr := r.Client.Get(ctx, client.ObjectKeyFromObject(desired), &existing); getErr != nil { + return fmt.Errorf("inspect colliding LMCache NodeLocal server Pod %s/%s: %w", desired.Namespace, desired.Name, getErr) + } + if metav1.IsControlledBy(&existing, backend) && existing.DeletionTimestamp != nil { + continue + } + return fmt.Errorf("LMCache NodeLocal server Pod name %s/%s is already occupied by another object", desired.Namespace, desired.Name) + } + return fmt.Errorf("create LMCache NodeLocal server Pod %s/%s: %w", desired.Namespace, desired.Name, err) + } + } + return nil +} + +func nodeLocalServerHasShmIdentity(pod *corev1.Pod, want string) bool { + if pod == nil || want == "" || pod.Annotations[enginebinding.AnnotationNodeLocalShmName] != want { + return false + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name != lmCacheMPServerStatusContainerName { + continue + } + args := pod.Spec.Containers[i].Args + for j := 0; j+1 < len(args); j++ { + if args[j] == "--shm-name" && args[j+1] == want { + return true + } + } + } + return false +} + +// nodeLocalEngineDemand returns one deterministic source engine per active +// node. The webhook-authored backend name+UID pair is required so overlapping +// selectors cannot make two CacheBackends provision servers for one engine. +// Generation is intentionally not required for lifecycle demand: after a +// CacheBackend update, existing engines remain a reason to keep their node's +// server alive while status reports that those immutable Pods need recreation. +func (r *CacheBackendReconciler) nodeLocalEngineDemand(ctx context.Context, backend *cachev1alpha1.CacheBackend) (map[string]*corev1.Pod, error) { + out := map[string]*corev1.Pod{} + selector := backend.Spec.EngineSelector + if selector == nil || len(selector.MatchLabels) == 0 { + return out, nil + } + reader := client.Reader(r.APIReader) + if reader == nil { + reader = r.Client + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, + client.InNamespace(backend.Namespace), + client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(selector.MatchLabels)}, + ); err != nil { + return nil, fmt.Errorf("list selected engines for NodeLocal CacheBackend %s/%s: %w", backend.Namespace, backend.Name, err) + } + sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Name < pods.Items[j].Name }) + wantOwner := backend.Namespace + "/" + backend.Name + for i := range pods.Items { + pod := &pods.Items[i] + if pod.DeletionTimestamp != nil || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed || pod.Spec.NodeName == "" { + continue + } + if pod.Annotations[enginebinding.AnnotationInjectedBy] != wantOwner || + pod.Annotations[enginebinding.AnnotationInjectedByUID] != string(backend.UID) { + continue + } + if out[pod.Spec.NodeName] == nil { + out[pod.Spec.NodeName] = pod + } + } + return out, nil +} + +func (r *CacheBackendReconciler) cleanupLMCacheNodeLocalServerPods(ctx context.Context, backend *cachev1alpha1.CacheBackend) error { + if backend == nil || backend.UID == "" { + return nil + } + var pods corev1.PodList + if err := r.Client.List(ctx, &pods, + client.InNamespace(backend.Namespace), + client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + enginebinding.LabelCacheBackendUID: string(backend.UID), + }, + ); err != nil { + return fmt.Errorf("list obsolete LMCache NodeLocal server Pods for %s/%s: %w", backend.Namespace, backend.Name, err) + } + for i := range pods.Items { + pod := &pods.Items[i] + if !metav1.IsControlledBy(pod, backend) || pod.DeletionTimestamp != nil { + continue + } + if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete LMCache NodeLocal server Pod %s/%s: %w", pod.Namespace, pod.Name, err) + } + } + return nil +} diff --git a/internal/controller/cachebackend_mp_lifecycle_test.go b/internal/controller/cachebackend_mp_lifecycle_test.go index d3ba43c2..7d2a287b 100644 --- a/internal/controller/cachebackend_mp_lifecycle_test.go +++ b/internal/controller/cachebackend_mp_lifecycle_test.go @@ -7,16 +7,350 @@ package controller import ( "context" "testing" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" + "github.com/cachebox-project/inference-cache/internal/enginebinding" ) +func nodeLocalBackend(name, namespace string) *cachev1alpha1.CacheBackend { + backend := lmcacheBackend(name, namespace) + backend.UID = types.UID("11111111-2222-3333-4444-555555555555") + backend.Spec.RemoteStorage = nil + backend.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}} + backend.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + backend.Spec.LMCache.PodLocal = nil + backend.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + IdleRetentionSeconds: 300, + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, HTTPPort: 18080, + L1Capacity: resource.MustParse("4Gi"), MaxGPUWorkers: 4, MaxCPUWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + } + return backend +} + +func nodeLocalEngine(backend *cachev1alpha1.CacheBackend, name, node string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: backend.Namespace, Labels: map[string]string{"app": "engine"}, Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: backend.Namespace + "/" + backend.Name, + enginebinding.AnnotationInjectedByUID: string(backend.UID), + enginebinding.AnnotationInjectedGeneration: "1", + }}, + Spec: corev1.PodSpec{NodeName: node, Containers: []corev1.Container{{Name: "vllm"}}}, + } +} + +func TestReconcileNodeLocalCreatesServersOnlyForEngineNodes(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + reconciler := newReconciler(newScheme(t), backend) + + reconcile(t, reconciler, backend.Name, backend.Namespace) + var empty corev1.PodList + if err := reconciler.List(context.Background(), &empty, client.InNamespace(backend.Namespace), client.MatchingLabels{enginebinding.LabelLMCacheNodeLocalServer: "true"}); err != nil { + t.Fatal(err) + } + if len(empty.Items) != 0 { + t.Fatalf("servers without scheduled engines = %d, want 0", len(empty.Items)) + } + + engineA := nodeLocalEngine(backend, "engine-a", "node-a") + engineB := nodeLocalEngine(backend, "engine-b", "node-a") + engineC := nodeLocalEngine(backend, "engine-c", "node-b") + for _, engine := range []*corev1.Pod{engineA, engineB, engineC} { + if err := reconciler.Create(context.Background(), engine); err != nil { + t.Fatalf("create engine %s: %v", engine.Name, err) + } + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + var servers corev1.PodList + if err := reconciler.List(context.Background(), &servers, client.InNamespace(backend.Namespace), client.MatchingLabels{enginebinding.LabelLMCacheNodeLocalServer: "true"}); err != nil { + t.Fatal(err) + } + if len(servers.Items) != 2 { + t.Fatalf("server Pods = %d, want one per distinct engine node", len(servers.Items)) + } + for i := range servers.Items { + server := &servers.Items[i] + if !metav1.IsControlledBy(server, backend) || !server.Spec.HostNetwork || server.Spec.NodeName != "" { + t.Fatalf("server ownership/placement = owner:%v hostNetwork:%v nodeName:%q", server.OwnerReferences, server.Spec.HostNetwork, server.Spec.NodeName) + } + target := server.Annotations[enginebinding.AnnotationNodeLocalTargetNode] + if target != "node-a" && target != "node-b" { + t.Fatalf("target node = %q", target) + } + if server.Name != builtinruntime.NodeLocalServerPodName(backend.Name, target) { + t.Fatalf("server name = %q, target %q", server.Name, target) + } + } + key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), key, &corev1.Service{}); !apierrors.IsNotFound(err) { + t.Fatalf("NodeLocal MP Service lookup = %v, want NotFound", err) + } + if err := reconciler.Get(context.Background(), key, &appsv1.Deployment{}); !apierrors.IsNotFound(err) { + t.Fatalf("host-only NodeLocal provider Deployment lookup = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalRetainsIdleServerAndReusesItUntilExpiry(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engineA := nodeLocalEngine(backend, "engine-a", "node-a") + engineB := nodeLocalEngine(backend, "engine-b", "node-a") + reconciler := newReconciler(newScheme(t), backend, engineA, engineB) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + serverKey := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Delete(context.Background(), engineA); err != nil { + t.Fatalf("delete first engine: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); err != nil { + t.Fatalf("shared server removed while one engine remained: %v", err) + } + + if err := reconciler.Delete(context.Background(), engineB); err != nil { + t.Fatalf("delete last engine: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + var idle corev1.Pod + if err := reconciler.Get(context.Background(), serverKey, &idle); err != nil { + t.Fatalf("server was not retained after last engine left: %v", err) + } + idleSince := idle.Annotations[enginebinding.AnnotationNodeLocalIdleSince] + if _, err := time.Parse(time.RFC3339Nano, idleSince); err != nil { + t.Fatalf("idle-since annotation = %q: %v", idleSince, err) + } + originalUID := idle.UID + + replacement := nodeLocalEngine(backend, "engine-c", "node-a") + if err := reconciler.Create(context.Background(), replacement); err != nil { + t.Fatalf("create replacement engine: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + var reused corev1.Pod + if err := reconciler.Get(context.Background(), serverKey, &reused); err != nil { + t.Fatalf("get reused server: %v", err) + } + if reused.UID != originalUID { + t.Fatalf("server UID changed during idle reuse: got %q, want %q", reused.UID, originalUID) + } + if _, found := reused.Annotations[enginebinding.AnnotationNodeLocalIdleSince]; found { + t.Fatalf("idle marker was not removed after demand returned: %+v", reused.Annotations) + } + + if err := reconciler.Delete(context.Background(), replacement); err != nil { + t.Fatalf("delete replacement engine: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), serverKey, &idle); err != nil { + t.Fatalf("get second idle server: %v", err) + } + before := idle.DeepCopy() + idle.Annotations[enginebinding.AnnotationNodeLocalIdleSince] = time.Now().Add(-301 * time.Second).UTC().Format(time.RFC3339Nano) + if err := reconciler.Patch(context.Background(), &idle, client.MergeFrom(before)); err != nil { + t.Fatalf("age idle marker: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("server after idle retention expired = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalZeroIdleRetentionDeletesImmediately(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 0 + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + if err := reconciler.Delete(context.Background(), engine); err != nil { + t.Fatalf("delete engine: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + serverKey := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), serverKey, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("server with zero idle retention = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalIgnoresSelectorMatchOwnedByAnotherBackend(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + engine.Annotations[enginebinding.AnnotationInjectedBy] = "ns1/other-cache" + engine.Annotations[enginebinding.AnnotationInjectedByUID] = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + var servers corev1.PodList + if err := reconciler.List(context.Background(), &servers, client.InNamespace(backend.Namespace), client.MatchingLabels{ + enginebinding.LabelLMCacheNodeLocalServer: "true", + }); err != nil { + t.Fatal(err) + } + if len(servers.Items) != 0 { + t.Fatalf("cross-CacheBackend selector overlap provisioned %d servers", len(servers.Items)) + } +} + +func TestReconcileNodeLocalRejectsOccupiedServerPodName(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + foreign := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace, + }, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "foreign"}}}} + reconciler := newReconciler(newScheme(t), backend, engine, foreign) + if err := reconciler.reconcileLMCacheNodeLocalServerPods(context.Background(), backend, nil); err == nil { + t.Fatal("occupied deterministic server Pod name was accepted") + } +} + +func TestReconcileNodeLocalManagedRedisKeepsLifecyclesIndependent(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + backend.Spec.RemoteStorage = lmcacheBackend("fixture", "ns1").Spec.RemoteStorage.DeepCopy() + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + + reconcile(t, reconciler, backend.Name, backend.Namespace) + + key := types.NamespacedName{Name: backend.Name, Namespace: backend.Namespace} + for kind, object := range map[string]client.Object{ + "NodeLocal server": &corev1.Pod{}, + "Redis Deployment": &appsv1.Deployment{}, + "Redis Service": &corev1.Service{}, + } { + objectKey := key + if kind == "NodeLocal server" { + objectKey.Name = builtinruntime.NodeLocalServerPodName(backend.Name, "node-a") + } + if err := reconciler.Get(context.Background(), objectKey, object); err != nil { + t.Fatalf("get %s: %v", kind, err) + } + } +} + +func TestReconcileNodeLocalToPodLocalDeletesServerPods(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + live := getBackend(t, reconciler, backend.Name, backend.Namespace) + podLocal := lmcacheBackend("fixture", "ns1").Spec.LMCache.PodLocal.DeepCopy() + live.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyPodLocal + live.Spec.LMCache.NodeLocal = nil + live.Spec.LMCache.PodLocal = podLocal + if err := reconciler.Update(context.Background(), live); err != nil { + t.Fatalf("update backend to PodLocal: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + key := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), key, &corev1.Pod{}); !apierrors.IsNotFound(err) { + t.Fatalf("server Pod after PodLocal transition = %v, want NotFound", err) + } +} + +func TestReconcileNodeLocalReplacesServerOnBackendGenerationChange(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + live := getBackend(t, reconciler, backend.Name, backend.Namespace) + live.Spec.LMCache.NodeLocal.Server.HTTPPort = 18081 + live.Generation++ // fake client does not emulate apiserver generation bumps + if err := reconciler.Update(context.Background(), live); err != nil { + t.Fatalf("update NodeLocal backend: %v", err) + } + reconcile(t, reconciler, backend.Name, backend.Namespace) + + var server corev1.Pod + key := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + if err := reconciler.Get(context.Background(), key, &server); err != nil { + t.Fatalf("get updated server Pod: %v", err) + } + ports := server.Spec.Containers[0].Ports + if len(ports) != 2 || ports[1].HostPort != 18081 { + t.Fatalf("updated server ports = %+v", ports) + } +} + +func TestReconcileNodeLocalReplacesServerMissingUIDScopedShmIdentity(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + key := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + var old corev1.Pod + if err := reconciler.Get(context.Background(), key, &old); err != nil { + t.Fatalf("get original server Pod: %v", err) + } + if err := reconciler.Delete(context.Background(), &old); err != nil { + t.Fatalf("delete original server Pod: %v", err) + } + delete(old.Annotations, enginebinding.AnnotationNodeLocalShmName) + old.ResourceVersion = "" + old.UID = "" + old.CreationTimestamp = metav1.Time{} + args := old.Spec.Containers[0].Args + for i := 0; i+1 < len(args); i++ { + if args[i] == "--shm-name" { + old.Spec.Containers[0].Args = append(args[:i], args[i+2:]...) + break + } + } + if err := reconciler.Create(context.Background(), &old); err != nil { + t.Fatalf("create legacy server Pod: %v", err) + } + + reconcile(t, reconciler, backend.Name, backend.Namespace) + var replaced corev1.Pod + if err := reconciler.Get(context.Background(), key, &replaced); err != nil { + t.Fatalf("get replacement server Pod: %v", err) + } + want, err := builtinruntime.NodeLocalServerShmName(backend) + if err != nil { + t.Fatal(err) + } + if !nodeLocalServerHasShmIdentity(&replaced, want) { + t.Fatalf("replacement server lacks UID-scoped shm identity: annotations=%v args=%v", replaced.Annotations, replaced.Spec.Containers[0].Args) + } +} + +func TestCleanupNodeLocalPreservesUnownedServerPod(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foreign", Namespace: backend.Namespace, Labels: map[string]string{ + enginebinding.LabelLMCacheNodeLocalServer: "true", enginebinding.LabelCacheBackendUID: string(backend.UID), + }}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "foreign"}}}} + reconciler := newReconciler(newScheme(t), backend, pod) + if err := reconciler.cleanupLMCacheNodeLocalServerPods(context.Background(), backend); err != nil { + t.Fatalf("cleanup unowned server Pod: %v", err) + } + if err := reconciler.Get(context.Background(), client.ObjectKeyFromObject(pod), &corev1.Pod{}); err != nil { + t.Fatalf("unowned server Pod was removed: %v", err) + } + if err := reconciler.cleanupLMCacheNodeLocalServerPods(context.Background(), nil); err != nil { + t.Fatalf("nil backend cleanup: %v", err) + } +} + func TestReconcileManagedRedisCreatesSingletonWorkload(t *testing.T) { backend := lmcacheBackend("cache", "ns1") reconciler := newReconciler(newScheme(t), backend) diff --git a/internal/controller/cachebackend_nodelocal_integration_test.go b/internal/controller/cachebackend_nodelocal_integration_test.go new file mode 100644 index 00000000..a067d220 --- /dev/null +++ b/internal/controller/cachebackend_nodelocal_integration_test.go @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package controller + +import ( + "context" + "strings" + "testing" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" + "github.com/cachebox-project/inference-cache/internal/enginebinding" +) + +// TestIntegrationCacheBackendNodeLocalServerPod exercises the engine-demanded +// child lifecycle against a real apiserver. Envtest has no scheduler or kubelet, +// so same-node health is GPU tested separately; this pins CREATE, exact-node +// affinity, UPDATE, ownership, and topology cleanup. +func TestIntegrationCacheBackendNodeLocalServerPod(t *testing.T) { + skipWithoutEnvtest(t) + k8s, scheme, _ := startEnv(t) + r := &CacheBackendReconciler{Client: k8s, APIReader: k8s, Scheme: scheme, Log: logr.Discard()} + ctx := context.Background() + ns := freshNS(t, k8s) + backend := nodeLocalBackend("node-cache", ns) + backend.UID = "" + if err := k8s.Create(ctx, backend); err != nil { + t.Fatalf("create NodeLocal CacheBackend: %v", err) + } + key := client.ObjectKey{Name: backend.Name, Namespace: ns} + var live cachev1alpha1.CacheBackend + if err := k8s.Get(ctx, key, &live); err != nil { + t.Fatalf("get NodeLocal CacheBackend: %v", err) + } + engine := nodeLocalEngine(&live, "engine-a", "node-a") + engine.Spec.Containers[0].Image = "example.invalid/engine:test" + if err := k8s.Create(ctx, engine); err != nil { + t.Fatalf("create scheduled engine: %v", err) + } + reconcile(t, r, backend.Name, ns) + + serverKey := client.ObjectKey{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: ns} + var server corev1.Pod + if err := k8s.Get(ctx, serverKey, &server); err != nil { + t.Fatalf("get NodeLocal server Pod: %v", err) + } + if !server.Spec.HostNetwork || server.Spec.HostIPC || server.Spec.DNSPolicy != "ClusterFirstWithHostNet" || server.Spec.NodeName != "" { + t.Fatalf("NodeLocal host/scheduler boundary = %+v", server.Spec) + } + if server.Spec.Affinity == nil || server.Spec.Affinity.NodeAffinity == nil { + t.Fatalf("server exact-node affinity missing: %+v", server.Spec.Affinity) + } + for _, port := range server.Spec.Containers[0].Ports { + if port.HostPort != port.ContainerPort || port.HostPort == 0 { + t.Fatalf("listener is not declared as hostPort: %+v", port) + } + } + if got := server.Labels[enginebinding.LabelCacheBackendUID]; got == "" { + t.Fatal("server Pod is missing CacheBackend UID identity") + } + wantShmName, err := builtinruntime.NodeLocalServerShmName(&live) + if err != nil { + t.Fatal(err) + } + if !nodeLocalServerHasShmIdentity(&server, wantShmName) { + t.Fatalf("server Pod lacks UID-scoped shm identity: annotations=%v args=%v", server.Annotations, server.Spec.Containers[0].Args) + } + + if err := k8s.Get(ctx, key, &live); err != nil { + t.Fatalf("get NodeLocal CacheBackend: %v", err) + } + live.Spec.LMCache.NodeLocal.Server.Port = 16556 + if err := k8s.Update(ctx, &live); err != nil { + t.Fatalf("update NodeLocal CacheBackend: %v", err) + } + reconcile(t, r, live.Name, live.Namespace) + if err := k8s.Get(ctx, serverKey, &server); err != nil { + t.Fatalf("get updated NodeLocal server Pod: %v", err) + } + if args := strings.Join(server.Spec.Containers[0].Args, " "); !strings.Contains(args, "--port 16556") { + t.Fatalf("server Pod args did not update: %s", args) + } + + if err := k8s.Get(ctx, key, &live); err != nil { + t.Fatalf("refresh NodeLocal CacheBackend: %v", err) + } + live.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyPodLocal + live.Spec.LMCache.PodLocal = lmcacheBackend("podlocal-shape", ns).Spec.LMCache.PodLocal.DeepCopy() + live.Spec.LMCache.NodeLocal = nil + if err := k8s.Update(ctx, &live); err != nil { + t.Fatalf("switch NodeLocal to PodLocal: %v", err) + } + reconcile(t, r, live.Name, live.Namespace) + if err := k8s.Get(ctx, serverKey, &server); !apierrors.IsNotFound(err) { + t.Fatalf("server Pod after PodLocal transition = %v, want NotFound", err) + } +} diff --git a/internal/controller/cachebackend_reconciler.go b/internal/controller/cachebackend_reconciler.go index 3a9897af..b5efe5de 100644 --- a/internal/controller/cachebackend_reconciler.go +++ b/internal/controller/cachebackend_reconciler.go @@ -6,19 +6,25 @@ package controller import ( "context" + "strings" + "time" + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/events" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" - "time" + ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" ) // DefaultMatchedEnginePodsRequeueInterval is the steady-state cadence at @@ -132,7 +138,7 @@ func (r *CacheBackendReconciler) matchedEnginePodsChurnRequeueInterval() time.Du // +kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get // +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;patch;delete // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch @@ -179,17 +185,16 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request // transient List/Patch errors so it never escalates a transient // apiserver hiccup into a Reconcile error. matchedRefresh := r.refreshMatchedEnginePods(ctx, &backend) - // Typed LMCache PodLocal health comes from the injected native sidecars in - // engine Pod status, independently from managed/external Redis readiness. + // Typed LMCache health comes from PodLocal native sidecars or NodeLocal + // on-demand server Pods and same-node engine coverage, independently from + // Redis readiness. r.refreshLMCacheMPConnectorStatus(ctx, &backend) // Self-requeue when there's matchedEnginePods work to keep doing on // the next tick: // - // - A non-empty engineSelector is configured. The cadence tracks - // pod birth/death between unrelated reconcile triggers. We - // deliberately don't Watch Pods (see refreshMatchedEnginePods - // godoc); the periodic self-requeue gives a bounded staleness - // without the watch's overhead. + // - A non-empty engineSelector is configured. Pod watches provide the + // normal trigger; this cadence is a bounded-staleness fallback for + // missed events and selector/annotation transitions. // - The selector is gone but status.matchedEnginePods is still // populated. That's the operator-just-removed-the-selector + // clear-patch-failed case: without a requeue the stale printer- @@ -229,20 +234,17 @@ func (r *CacheBackendReconciler) Reconcile(ctx context.Context, req ctrl.Request return result, err } -// SetupWithManager sets up the controller with the Manager. Owns(Deployment) -// guarantees that a child's status flipping (e.g. AvailableReplicas dropping -// to zero) re-triggers a Reconcile so emitTransitionEvents observes the -// change. +// SetupWithManager sets up the controller with the Manager. Owned provider +// workload changes and mapped engine/NodeLocal-server Pod changes re-trigger a +// Reconcile so lifecycle, coverage, and transition Events track live state. func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { if r.Recorder == nil { r.Recorder = mgr.GetEventRecorder("cachebackend-controller") } if r.APIReader == nil { - // Default to the manager's uncached APIReader so production - // wiring doesn't have to thread it explicitly, AND envtest - // integration tests that boot a real manager still skip the - // Pod informer per the locked design (the test setup just - // passes Client, not APIReader). + // Use the uncached reader for demand/status snapshots. The Pod watch is + // the event trigger; an authoritative list avoids acting on an informer + // snapshot that has not yet observed the scheduling or deletion event. r.APIReader = mgr.GetAPIReader() } return ctrl.NewControllerManagedBy(mgr). @@ -255,5 +257,26 @@ func (r *CacheBackendReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&cachev1alpha1.CacheBackend{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). + Watches(&corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(cacheBackendRequestForPod)). Complete(r) } + +// cacheBackendRequestForPod maps controller-owned NodeLocal server Pods and +// webhook-injected engine Pods back to their CacheBackend. Node assignment, +// readiness, deletion, and server scheduling updates therefore drive on-demand +// lifecycle and status immediately. +func cacheBackendRequestForPod(_ context.Context, obj client.Object) []ctrlreconcile.Request { + if obj == nil { + return nil + } + if owner := metav1.GetControllerOf(obj); owner != nil && owner.Kind == "CacheBackend" && owner.APIVersion == cachev1alpha1.GroupVersion.String() { + return []ctrlreconcile.Request{{NamespacedName: client.ObjectKey{Namespace: obj.GetNamespace(), Name: owner.Name}}} + } + annotations := obj.GetAnnotations() + ref := annotations[enginebinding.AnnotationInjectedBy] + if !validCacheBackendRef(ref) || annotations[enginebinding.AnnotationInjectedByUID] == "" { + return nil + } + parts := strings.SplitN(ref, "/", 2) + return []ctrlreconcile.Request{{NamespacedName: client.ObjectKey{Namespace: parts[0], Name: parts[1]}}} +} diff --git a/internal/controller/cachebackend_reconciler_test.go b/internal/controller/cachebackend_reconciler_test.go index c10bd9a5..c70b7613 100644 --- a/internal/controller/cachebackend_reconciler_test.go +++ b/internal/controller/cachebackend_reconciler_test.go @@ -8,6 +8,7 @@ import ( "context" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" builtinadapters "github.com/cachebox-project/inference-cache/internal/adapters/builtin" + "github.com/cachebox-project/inference-cache/internal/enginebinding" backendadapter "github.com/cachebox-project/inference-cache/pkg/adapters/backend" adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" "github.com/go-logr/logr" @@ -18,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -73,6 +75,46 @@ func configureTestRegistries(r *CacheBackendReconciler) { } } +func TestCacheBackendRequestForPod(t *testing.T) { + tests := []struct { + name string + pod *corev1.Pod + want types.NamespacedName + }{ + { + name: "controller-owned server", + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", OwnerReferences: []metav1.OwnerReference{{ + APIVersion: cachev1alpha1.GroupVersion.String(), Kind: "CacheBackend", Name: "cache", Controller: ptr.To(true), + }}}}, + want: types.NamespacedName{Namespace: "team-a", Name: "cache"}, + }, + { + name: "injected engine", + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: "team-a/cache", enginebinding.AnnotationInjectedByUID: "uid", + }}}, + want: types.NamespacedName{Namespace: "team-a", Name: "cache"}, + }, + {name: "unverified engine", pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "team-a", Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: "team-a/cache", + }}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cacheBackendRequestForPod(context.Background(), tt.pod) + if tt.want.Name == "" { + if len(got) != 0 { + t.Fatalf("requests = %+v, want none", got) + } + return + } + if len(got) != 1 || got[0].NamespacedName != tt.want { + t.Fatalf("requests = %+v, want %s", got, tt.want) + } + }) + } +} + func setupTestCacheBackendReconciler(mgr ctrl.Manager, r *CacheBackendReconciler) error { configureTestRegistries(r) return r.SetupWithManager(mgr) diff --git a/internal/controller/cachebackend_serverless.go b/internal/controller/cachebackend_serverless.go index 18bd18d9..ebb84c5e 100644 --- a/internal/controller/cachebackend_serverless.go +++ b/internal/controller/cachebackend_serverless.go @@ -111,7 +111,7 @@ func (r *CacheBackendReconciler) reconcileExternal(ctx context.Context, backend } remoteStatus := readyStatus setRemoteStorageStatus(backend, endpoint, remoteStatus, remoteReason, remoteMessage, backend.Generation) - if isTypedLMCachePodLocal(backend) { + if isTypedLMCacheMP(backend) { readyStatus, readyReason, readyMsg = lmCacheMPReadyBase(backend, remoteStatus, remoteReason, remoteMessage) } progressingStatus, progressingReason, progressingMessage := progressingFromReady(readyStatus, readyReason, readyMsg) @@ -263,7 +263,7 @@ func (r *CacheBackendReconciler) reconcileServerless(ctx context.Context, backen // available merely because it has no separately managed workload: wait // for the connector observation, otherwise a Pod that appears after the // timeout window would be declared NoKVEventsObserved immediately. - canAnchorAvailability := eventsOnly || !isTypedLMCachePodLocal(backend) || baseStatus == metav1.ConditionTrue + canAnchorAvailability := eventsOnly || !isTypedLMCacheMP(backend) || baseStatus == metav1.ConditionTrue if transitionedFromServerMode && !canAnchorAvailability { backend.Status.FirstAvailableAt = nil } else if canAnchorAvailability && (backend.Status.FirstAvailableAt == nil || transitionedFromServerMode) { diff --git a/internal/controller/cachebackend_status.go b/internal/controller/cachebackend_status.go index dc3361bf..5a55bbc1 100644 --- a/internal/controller/cachebackend_status.go +++ b/internal/controller/cachebackend_status.go @@ -172,7 +172,7 @@ func (r *CacheBackendReconciler) updateManagedStatus(ctx context.Context, backen } else { remoteReason = reasonRemoteStorageUnavailable } - if isTypedLMCachePodLocal(backend) { + if isTypedLMCacheMP(backend) { readyStatus, reason, message = lmCacheMPReadyBase(backend, remoteStatus, remoteReason, remoteMessage) } // Resolve the stable timeout anchor: the latched FirstAvailableAt, or — the diff --git a/internal/controller/cachebackend_workload.go b/internal/controller/cachebackend_workload.go index 9b45073d..b09c29dd 100644 --- a/internal/controller/cachebackend_workload.go +++ b/internal/controller/cachebackend_workload.go @@ -335,6 +335,7 @@ func reconcileManagedContainer(live *corev1.PodSpec, desired *corev1.PodSpec) { live.Containers[j].ReadinessProbe = want.ReadinessProbe live.Containers[j].LivenessProbe = want.LivenessProbe live.Containers[j].StartupProbe = want.StartupProbe + live.Containers[j].SecurityContext = want.SecurityContext matched = true break } diff --git a/internal/controller/cacheindex_controller.go b/internal/controller/cacheindex_controller.go index 3ddd29bd..5c77c665 100644 --- a/internal/controller/cacheindex_controller.go +++ b/internal/controller/cacheindex_controller.go @@ -259,13 +259,13 @@ func (p *CacheIndexPoller) reconcileTenantStatuses(ctx context.Context, snap con // replica_id=, tenant_id=. The CacheBackend // points at the same engine pod via spec.engineSelector.matchLabels. // So: look up the engine pod by (replica.Tenant, replica.ReplicaID), -// then attribute to the FIRST in-namespace CacheBackend whose -// EngineSelector matches the pod's labels — mirroring the pod -// webhook's "first-match wins" engine-wiring rule (see -// internal/webhook/pod/podinjector.go's selectCacheBackend). Two -// backends with overlapping selectors must agree on which one owns -// the pod, or status will disagree with what the engine was actually -// wired to. +// The webhook's inferencecache.io/injected-by annotation is the +// authoritative owner. For manually attached subscriber pods without that +// annotation, fall back to the first matching +// in-namespace CacheBackend in metadata.name order. New CacheBackend +// admission rejects overlapping selectors and fresh Pod admission denies +// an ambiguous match; the fallback is attribution only and never chooses a +// connector owner for an admitted engine. // - A replica with no engine pod found (pod was deleted between events // and now) is dropped — its contributions only show up in the // cluster-wide CacheIndex. @@ -313,10 +313,9 @@ func (p *CacheIndexPoller) refreshCacheBackendParticipation(ctx context.Context, } backendsByNS[cb.Namespace] = append(backendsByNS[cb.Namespace], i) } - // Sort the per-namespace backend lists by metadata.name. This gives - // "first match" a deterministic meaning across poller restarts and - // makes the selector-fallback result independent of apiserver List - // ordering — operators who rely on the fallback can predict the winner. + // Sort the per-namespace backend lists by metadata.name so the historical + // selector fallback is stable across poller restarts and independent of + // apiserver List ordering. for ns := range backendsByNS { idxs := backendsByNS[ns] sort.Slice(idxs, func(a, b int) bool { @@ -504,11 +503,11 @@ func matchLabelsSelects(want, have map[string]string) bool { // the authoritative signal: it records the CacheBackend the webhook // actually wired the engine to. Annotation in another namespace is // ignored (cross-namespace attribution would be misleading). -// 2. Fallback for pods that bypassed the webhook (manual sidecar, opt-out -// annotation): iterate the namespace's CacheBackends sorted by name and -// take the first whose EngineSelector matches the pod's labels — -// mirroring the webhook's first-match rule but ordered deterministically -// by name so the poller is stable across restarts. +// 2. Fallback for historical pods or pods that bypassed the webhook (manual +// sidecar, opt-out annotation): iterate the namespace's CacheBackends +// sorted by name and take the first whose EngineSelector matches the +// pod's labels. Admission prevents this ambiguity for new objects; the +// ordering only keeps legacy attribution stable across restarts. func (p *CacheIndexPoller) attributePod(pod *corev1.Pod, nsBackends []int, byNSName map[types.NamespacedName]int, items []cachev1alpha1.CacheBackend) int { if raw := pod.Annotations[enginebinding.AnnotationInjectedBy]; raw != "" { ns, name, ok := strings.Cut(raw, "/") diff --git a/internal/controller/cacheindex_controller_test.go b/internal/controller/cacheindex_controller_test.go index f0cb5aee..0694c413 100644 --- a/internal/controller/cacheindex_controller_test.go +++ b/internal/controller/cacheindex_controller_test.go @@ -1117,13 +1117,13 @@ func TestRefreshScrapeFailureDoesNotClearParticipation(t *testing.T) { } } -// TestRefreshOverlappingSelectorsFirstNameWins: two CacheBackends with -// overlapping EngineSelector that both match the same engine pod — the -// poller must attribute only to the deterministic "first by name" backend, -// mirroring the webhook's one-pod-one-backend wiring rule. Attributing to -// both would tell operators a backend is contributing when its engine was -// actually wired to the other backend's endpoint. -func TestRefreshOverlappingSelectorsFirstNameWins(t *testing.T) { +// TestRefreshManualOverlappingSelectorsFallbackIsDeterministic covers +// manually created state that bypassed admission. Without an authoritative +// injected-by annotation, the poller +// attributes to the deterministic first backend by name rather than claiming +// both backends. Fresh CacheBackend and Pod admission cannot create this +// ambiguous state. +func TestRefreshManualOverlappingSelectorsFallbackIsDeterministic(t *testing.T) { cbAlpha := cbFixture("alpha", "default", map[string]string{"app": "vllm"}) cbBeta := cbFixture("beta", "default", map[string]string{"app": "vllm", "model": "llama"}) podMatch := enginePod("vllm-0", "default", map[string]string{"app": "vllm", "model": "llama"}) diff --git a/internal/controller/contract_coverage_sweep_test.go b/internal/controller/contract_coverage_sweep_test.go index d76176c1..90239c02 100644 --- a/internal/controller/contract_coverage_sweep_test.go +++ b/internal/controller/contract_coverage_sweep_test.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -37,80 +36,22 @@ import ( podwebhook "github.com/cachebox-project/inference-cache/internal/webhook/pod" ) -// TestWebhookPollerSelectorFallbackAgreement is the load-bearing -// agreement test for the selector fallback. The pod webhook -// (internal/webhook/pod.selectCacheBackend) and the CacheIndex poller -// (attributePod) BOTH sort CacheBackends by metadata.name ascending and -// take the first selector match. They are two independent code paths -// against the same contract: when an engine pod matches more than one -// CacheBackend, both surfaces must converge on the same backend, or the -// webhook will wire the engine to one backend's endpoint while the -// poller's status writer attributes its participation to another. Each -// surface has its own first-by-name test already; this test is the one -// place that asserts the two surfaces agree. A future rename of the -// sort key (or accidental switch to creationTimestamp) on either side -// must fail HERE. -func TestWebhookPollerSelectorFallbackAgreement(t *testing.T) { +// TestWebhookRejectsSelectorAmbiguity is the runtime backstop for historical +// CacheBackends and concurrent CREATE races that bypass the normal +// cross-object admission check. A fresh engine must never be wired to an +// arbitrary first-by-name cache trust domain. +func TestWebhookRejectsSelectorAmbiguity(t *testing.T) { const ns = "agree-ns" labels := map[string]string{"app": "vllm"} - // Two CacheBackends with identical selectors. "alpha" should win on - // both surfaces (lexicographically before "zebra"). The two helpers - // produce different shapes — readyCacheBackendForSweep includes - // status.remoteStorage.endpoint (the webhook needs it for Redis) and cbFixture is - // selector-only (the poller's attribution doesn't depend on endpoint). cbAlphaWebhook := readyCacheBackendForSweep("alpha", ns, labels) cbZebraWebhook := readyCacheBackendForSweep("zebra", ns, labels) - - // Surface 1 — webhook: admit a fresh engine pod and read the - // injected-by annotation the handler stamps. The annotation records - // the CacheBackend the webhook actually wired the engine to. - annotated := runPodWebhookAndCaptureInjectedBy(t, ns, - cbAlphaWebhook, cbZebraWebhook, labels) - webhookChose, ok := parseInjectedByForSweep(annotated) - if !ok || webhookChose.Namespace != ns { - t.Fatalf("webhook annotation %q parsed to %+v; want namespace %q", annotated, webhookChose, ns) - } - - // Surface 2 — poller: feed the SAME backends + a same-shaped engine - // pod into the poller, with the annotation stripped so attribution - // falls through the selector path (not the annotation shortcut). The - // snapshot is keyed by the sidecar-identity convention - // (replica_id = pod_name, tenant_id = pod_namespace). The fallback's - // winner is observable as the backend whose - // status.indexParticipation.prefixCount is non-zero after the refresh. - cbAlphaPoller := cbFixture("alpha", ns, labels) - cbZebraPoller := cbFixture("zebra", ns, labels) - const podName = "engine-a" - pollerPod := enginePod(podName, ns, labels) // no injected-by annotation - var mu sync.Mutex - served := controlplaneapi.Snapshot{ - Replicas: []controlplaneapi.ReplicaSnapshot{ - {ReplicaID: podName, Tenant: ns, PrefixCount: 7, LastEventAt: time.Unix(1_700_000_000, 0).UTC()}, - }, - } - p, cl, srv := buildPollerWithFixtures(t, - []*cachev1alpha1.CacheBackend{cbAlphaPoller, cbZebraPoller}, - []*corev1.Pod{pollerPod}, - &served, &mu) - defer srv.Close() - if err := p.refresh(context.Background()); err != nil { - t.Fatalf("poller refresh: %v", err) - } - pollerChose := pickAttributedBackend(t, cl, []string{"alpha", "zebra"}, ns) - - // The actual cross-system agreement assertion: ONE rename on either - // surface (sort field, selector semantics, etc.) drives these apart. - if pollerChose.Name != webhookChose.Name || pollerChose.Namespace != webhookChose.Namespace { - t.Fatalf("webhook+poller fallback disagreement: webhook chose %s/%s, poller chose %s/%s", - webhookChose.Namespace, webhookChose.Name, - pollerChose.Namespace, pollerChose.Name) + resp := runPodWebhookForSweep(t, ns, cbAlphaWebhook, cbZebraWebhook, labels) + if resp.Allowed { + t.Fatalf("overlapping selectors admitted with patches %+v", resp.Patches) } - // Both must converge on lex-first ("alpha"). A pivot to a different - // sort policy must be a deliberate joint change, with the doc - // comments on both surfaces updated. - if webhookChose.Name != "alpha" { - t.Fatalf("both surfaces should converge on 'alpha' (first by name); both chose %q", webhookChose.Name) + if resp.Result == nil || !strings.Contains(resp.Result.Message, "multiple CacheBackends") { + t.Fatalf("ambiguity denial = %+v, want explicit multiple-CacheBackend message", resp.Result) } } @@ -324,8 +265,8 @@ func readyCacheBackendForSweep(name, namespace string, selector map[string]strin // would invoke Handle() at admission time without standing up a full // envtest — the agreement scenario doesn't need a real apiserver, just // the two surfaces' actual selection logic. -func runPodWebhookAndCaptureInjectedBy(t *testing.T, namespace string, - cb1, cb2 *cachev1alpha1.CacheBackend, podLabels map[string]string) string { +func runPodWebhookForSweep(t *testing.T, namespace string, + cb1, cb2 *cachev1alpha1.CacheBackend, podLabels map[string]string) admission.Response { t.Helper() scheme := runtime.NewScheme() if err := clientgoscheme.AddToScheme(scheme); err != nil { @@ -363,69 +304,5 @@ func runPodWebhookAndCaptureInjectedBy(t *testing.T, namespace string, }, } resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("webhook response not Allowed: %+v", resp.Result) - } - if len(resp.Patches) == 0 { - t.Fatal("expected JSON patches from webhook (overlapping selectors should produce a match)") - } - annotationEscaped := jsonPatchEscapeForSweep(podwebhook.AnnotationInjectedBy) - for _, p := range resp.Patches { - // The webhook can emit the annotation in two patch shapes: - // either an add of the whole annotations map, or an add of the - // single annotation key. Handle both rather than assuming one. - switch v := p.Value.(type) { - case map[string]any: - if s, ok := v[podwebhook.AnnotationInjectedBy].(string); ok && s != "" { - return s - } - case string: - if p.Path == "/metadata/annotations/"+annotationEscaped { - return v - } - } - } - t.Fatalf("webhook did not stamp %q annotation; patches = %+v", - podwebhook.AnnotationInjectedBy, resp.Patches) - return "" -} - -// pickAttributedBackend returns the (namespace, name) of the single -// CacheBackend in the supplied set whose status.indexParticipation is -// populated with a non-zero prefixCount. Fails the test if zero or more -// than one match — both surfaces must converge on exactly one winner. -func pickAttributedBackend(t *testing.T, cl client.Client, names []string, namespace string) types.NamespacedName { - t.Helper() - var winners []types.NamespacedName - for _, n := range names { - cb := getBackendDirect(t, cl, n, namespace) - if cb.Status.IndexParticipation != nil && cb.Status.IndexParticipation.PrefixCount > 0 { - winners = append(winners, types.NamespacedName{Namespace: namespace, Name: n}) - } - } - if len(winners) != 1 { - t.Fatalf("expected exactly one attributed backend; got %v", winners) - } - return winners[0] -} - -// parseInjectedByForSweep splits a "namespace/name" injected-by annotation -// into a NamespacedName. Returns ok=false on a malformed value so the -// caller can produce a context-specific failure message. -func parseInjectedByForSweep(value string) (types.NamespacedName, bool) { - ns, name, ok := strings.Cut(value, "/") - if !ok || ns == "" || name == "" { - return types.NamespacedName{}, false - } - return types.NamespacedName{Namespace: ns, Name: name}, true -} - -// jsonPatchEscapeForSweep is the JSON-Pointer escaping for "/" and "~" in -// JSON Patch paths (RFC 6901). Matches the helper of the same shape in -// the webhook tests so the agreement test can identify the annotation -// patch regardless of how controller-runtime renders it. -func jsonPatchEscapeForSweep(s string) string { - s = strings.ReplaceAll(s, "~", "~0") - s = strings.ReplaceAll(s, "/", "~1") - return s + return resp } diff --git a/internal/enginebinding/metadata.go b/internal/enginebinding/metadata.go index 6ca2d520..8a91fea0 100644 --- a/internal/enginebinding/metadata.go +++ b/internal/enginebinding/metadata.go @@ -20,6 +20,40 @@ const ( // LMCache PodMonitor. LabelLMCacheMPMetricsEnabled = "true" + // LabelLMCacheNodeLocalServer identifies a controller-owned NodeLocal MP + // server Pod. + LabelLMCacheNodeLocalServer = "inferencecache.io/lmcache-node-server" + + // LabelCacheBackendUID is the immutable, label-safe identity used to list + // one CacheBackend's NodeLocal server Pods. + LabelCacheBackendUID = "inferencecache.io/cache-backend-uid" + + // AnnotationNodeLocalOwner records the namespace/name of the CacheBackend + // whose NodeLocal server configuration the Pod carries. + AnnotationNodeLocalOwner = "inferencecache.io/node-local-owner" + + // AnnotationNodeLocalOwnerUID authenticates the name against the current + // CacheBackend UID so delete/recreate races cannot claim a stale server. + AnnotationNodeLocalOwnerUID = "inferencecache.io/node-local-owner-uid" + + // AnnotationNodeLocalGeneration records the exact CacheBackend generation + // rendered into a NodeLocal server Pod. + AnnotationNodeLocalGeneration = "inferencecache.io/node-local-generation" + + // AnnotationNodeLocalTargetNode records the engine-selected node for which + // this server Pod was rendered. The Pod still uses scheduler-bound exact + // node affinity rather than spec.nodeName so hostPort conflicts are checked. + AnnotationNodeLocalTargetNode = "inferencecache.io/node-local-target-node" + + // AnnotationNodeLocalShmName records the controller-derived POSIX shared + // memory object owned by this CacheBackend's NodeLocal server pool. + AnnotationNodeLocalShmName = "inferencecache.io/node-local-shm-name" + + // AnnotationNodeLocalIdleSince records when the final active engine left a + // node. The controller removes it when demand returns and deletes the server + // only after the configured idle-retention window expires. + AnnotationNodeLocalIdleSince = "inferencecache.io/node-local-idle-since" + // AnnotationSkip lets an operator explicitly opt a pod out of injection. AnnotationSkip = "inferencecache.io/skip-inject" diff --git a/internal/webhook/pod/envtest_integration_test.go b/internal/webhook/pod/envtest_integration_test.go index ef4fb23d..b9f5868c 100644 --- a/internal/webhook/pod/envtest_integration_test.go +++ b/internal/webhook/pod/envtest_integration_test.go @@ -444,6 +444,61 @@ func TestWebhookOnEnvtest_EndToEnd(t *testing.T) { if !envtestHasContainerEnvValue(&gotVLLMMP, "PYTHONHASHSEED", "0") { t.Fatalf("typed vLLM Pod is missing PYTHONHASHSEED=0: %+v", gotVLLMMP.Spec.Containers[0].Env) } + + // Typed vLLM NodeLocal smoke: the real apiserver must preserve hostPath + // /dev/shm, preserved inference-owned placement, the Downward API node address, and + // the blocking ownership gate. No PodLocal native sidecar may be present. + nodeLocalCB := &cachev1alpha1.CacheBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "vllm-node-mp", Namespace: ns}, + Spec: cachev1alpha1.CacheBackendSpec{ + Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, + Type: cachev1alpha1.CacheBackendTypeLMCache, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + "app": "vllm-node-mp-test", + }}, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyNodeLocal, + NodeLocal: &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + Port: 16555, HTTPPort: 18080, L1Capacity: resource.MustParse("1Gi"), MaxGPUWorkers: 2, MaxCPUWorkers: 2, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("2Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + }, + }, + }, + } + if err := mgr.GetClient().Create(ctx, nodeLocalCB); err != nil { + t.Fatalf("create typed vLLM NodeLocal CacheBackend: %v", err) + } + nodeLocalPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "vllm-node-mp-engine", Namespace: ns, Labels: map[string]string{"app": "vllm-node-mp-test"}}, + Spec: corev1.PodSpec{NodeSelector: map[string]string{"gpu-pool": "inference-owned"}, Containers: []corev1.Container{{ + Name: "vllm", Image: "vllm:connector-ready", Args: []string{"--model", "model-a", "--tensor-parallel-size=1"}, + }}}, + } + if err := mgr.GetClient().Create(ctx, nodeLocalPod); err != nil { + t.Fatalf("create typed vLLM NodeLocal Pod: %v", err) + } + var gotNodeLocal corev1.Pod + if err := mgr.GetAPIReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeLocalPod.Name}, &gotNodeLocal); err != nil { + t.Fatalf("get typed vLLM NodeLocal Pod: %v", err) + } + if envtestFindInitContainer(&gotNodeLocal, "lmcache-mp-server") != nil || envtestFindInitContainer(&gotNodeLocal, "lmcache-node-local-gate") == nil { + t.Fatalf("NodeLocal init containers = %+v", gotNodeLocal.Spec.InitContainers) + } + if len(gotNodeLocal.Spec.NodeSelector) != 1 || gotNodeLocal.Spec.NodeSelector["gpu-pool"] != "inference-owned" || gotNodeLocal.Spec.Affinity != nil { + t.Fatalf("NodeLocal mutated inference-owned placement: selector:%v affinity:%+v", gotNodeLocal.Spec.NodeSelector, gotNodeLocal.Spec.Affinity) + } + nodeConfig := envtestArgValue(gotNodeLocal.Spec.Containers[0].Args, "--kv-transfer-config") + if !strings.Contains(nodeConfig, `"lmcache.mp.host":"tcp://$(INFERENCECACHE_NODE_IP)"`) || + !envtestContainerHasFieldRef(&gotNodeLocal, "INFERENCECACHE_NODE_IP", "status.hostIP") { + t.Fatalf("NodeLocal node-derived engine wire = config:%q env:%+v", nodeConfig, gotNodeLocal.Spec.Containers[0].Env) + } } func envtestArgValue(args []string, flag string) string { @@ -470,6 +525,18 @@ func envtestHasContainerEnvValue(pod *corev1.Pod, name, value string) bool { return false } +func envtestContainerHasFieldRef(pod *corev1.Pod, name, path string) bool { + if pod == nil || len(pod.Spec.Containers) == 0 { + return false + } + for _, entry := range pod.Spec.Containers[0].Env { + if entry.Name == name && entry.ValueFrom != nil && entry.ValueFrom.FieldRef != nil && entry.ValueFrom.FieldRef.FieldPath == path { + return true + } + } + return false +} + // envtestFindContainer returns the container in pod with the given name, or // nil if absent. The non-envtest unit tests have a similarly named helper — // the two test files don't share state (envtest_integration_test.go skips diff --git a/internal/webhook/pod/podinjector.go b/internal/webhook/pod/podinjector.go index 35cd5daa..c9e3c8f1 100644 --- a/internal/webhook/pod/podinjector.go +++ b/internal/webhook/pod/podinjector.go @@ -7,6 +7,7 @@ package pod import ( "context" "encoding/json" + "errors" "fmt" "sort" "strconv" @@ -98,11 +99,13 @@ const InjectSkippedReasonSkipAnnotation = enginebinding.InjectSkippedReasonSkipA // +kubebuilder:rbac:groups=inferencecache.io,resources=cachebackends,verbs=get;list;watch // EngineInjector is the admission.Handler that injects LMCache engine -// configuration into user-provided engine pods. failurePolicy=ignore on the -// MutatingWebhookConfiguration AND a fail-open posture in the handler give a -// belt-and-suspenders guarantee: even if the controller is unreachable or -// the handler returns an error response, pod admission is never blocked. -// The cache is always an optimization, never a serving dependency. +// configuration into user-provided engine pods. Operational lookup/adapter +// failures remain fail-open because the cache is an optimization. A selector +// ambiguity is different: choosing one of several trust domains would be an +// unsafe mutation, so a live webhook explicitly denies that Pod. The +// MutatingWebhookConfiguration still uses failurePolicy=Ignore for webhook +// transport outages; CacheBackend admission prevents overlaps in the normal +// path and this denial is the defense for historical/concurrent conflicts. type EngineInjector struct { // Reader lists CacheBackends in the pod's namespace. Production wiring // passes the manager's APIReader (an uncached live client) — pod @@ -124,11 +127,10 @@ type EngineInjector struct { Log logr.Logger } -// Handle implements [admission.Handler]. Any rejection at this layer -// translates to admission.Allowed: a webhook error MUST NOT block pod -// admission (the cache is an optimization). The reason string carries -// enough context that an operator running `kubectl get events` can tell -// why a pod was admitted without injection. +// Handle implements [admission.Handler]. Operational failures translate to +// admission.Allowed because the cache is an optimization. Selector ambiguity +// is the deliberate exception: choosing a cache trust domain is unsafe, so a +// live webhook denies the Pod. The reason string explains either outcome. func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admission.Response { log := h.logger(ctx).WithValues( "namespace", req.Namespace, "name", req.Name, "uid", string(req.UID), @@ -161,6 +163,11 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi cache, err := h.selectCacheBackend(ctx, &pod) if err != nil { + var ambiguity *ambiguousCacheBackendMatchError + if errors.As(err, &ambiguity) { + log.Info("rejecting Pod: more than one CacheBackend engineSelector matches", "cachebackends", ambiguity.names) + return admission.Denied(err.Error()) + } log.V(1).Info("fail-open: backend lookup failed", "error", err.Error()) return failOpen(req, &pod, fmt.Sprintf("backend lookup failed (fail-open): %v", err)) } @@ -459,25 +466,12 @@ func (h *EngineInjector) Handle(ctx context.Context, req admission.Request) admi return resp } -// selectCacheBackend returns the first CacheBackend in pod.Namespace whose +// selectCacheBackend returns the only CacheBackend in pod.Namespace whose // Spec.EngineSelector.MatchLabels match pod.Labels, or nil when no backend -// claims the pod. Selecting "the first match" is deliberately simple for -// Phase 1: a future revision can grow a tie-break policy (e.g. an explicit -// `inferencecache.io/cachebackend: ` annotation), but the current rule -// matches the reconciler's "each CacheBackend owns its EngineSelector" -// contract — an operator running two backends in the same namespace whose -// selectors overlap is misconfigured, and the handler logs the chosen one -// so the ambiguity is observable. -// -// Iteration order is metadata.name ascending so the "first match" is -// deterministic across apiserver List cache states and is shared with the -// CacheIndex poller's annotation-fallback path (see -// internal/controller/cacheindex_controller.go's attributePod). The two -// surfaces MUST agree on which backend owns a given engine pod — the -// webhook stamps `inferencecache.io/injected-by` and the poller relies -// on it as the authoritative signal, but on overlapping-selector -// fallback both sides need to pick the same backend or status will -// disagree with what the engine was wired to. +// claims the pod. More than one match is rejected: silently choosing one would +// route an engine into an unintended cache trust domain. CacheBackend admission +// rejects overlapping selectors in the normal path; this runtime check covers +// objects admitted before that rule and concurrent CREATE races. // // A CacheBackend with no EngineSelector or with an empty MatchLabels map is // skipped: a "match everything" selector at admission time would silently @@ -497,6 +491,7 @@ func (h *EngineInjector) selectCacheBackend(ctx context.Context, pod *corev1.Pod return list.Items[idxs[a]].Name < list.Items[idxs[b]].Name }) podLabels := labels.Set(pod.Labels) + matches := make([]*cachev1alpha1.CacheBackend, 0, 1) for _, i := range idxs { cb := &list.Items[i] if cb.Spec.EngineSelector == nil || len(cb.Spec.EngineSelector.MatchLabels) == 0 { @@ -504,10 +499,30 @@ func (h *EngineInjector) selectCacheBackend(ctx context.Context, pod *corev1.Pod } sel := labels.SelectorFromSet(cb.Spec.EngineSelector.MatchLabels) if sel.Matches(podLabels) { - return cb, nil + matches = append(matches, cb) } } - return nil, nil + if len(matches) == 0 { + return nil, nil + } + if len(matches) == 1 { + return matches[0], nil + } + names := make([]string, len(matches)) + for i := range matches { + names[i] = matches[i].Name + } + return nil, &ambiguousCacheBackendMatchError{namespace: pod.Namespace, names: names} +} + +type ambiguousCacheBackendMatchError struct { + namespace string + names []string +} + +func (e *ambiguousCacheBackendMatchError) Error() string { + return fmt.Sprintf("Pod labels match multiple CacheBackends in namespace %q (%s); engine admission requires exactly one CacheBackend owner", + e.namespace, strings.Join(e.names, ", ")) } // SkipAnnotationOptsOut returns true when the value of [AnnotationSkip] diff --git a/internal/webhook/pod/podinjector_test.go b/internal/webhook/pod/podinjector_test.go index 229b41f7..af9579b1 100644 --- a/internal/webhook/pod/podinjector_test.go +++ b/internal/webhook/pod/podinjector_test.go @@ -323,6 +323,26 @@ func typedVLLMPodLocalBackend(name, namespace string, selector map[string]string return cb } +func typedVLLMNodeLocalBackend(name, namespace string, selector map[string]string) *cachev1alpha1.CacheBackend { + cb := typedVLLMPodLocalBackend(name, namespace, selector) + cb.Generation = 3 + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, HTTPPort: 18080, + L1Capacity: resource.MustParse("8Gi"), MaxGPUWorkers: 4, MaxCPUWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("9Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("9Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + } + return cb +} + func TestHandle_TypedVLLMWithoutCapabilityAnnotationsUsesDedicatedMPAdapter(t *testing.T) { const ns = "engines" cb := typedVLLMPodLocalBackend("primary", ns, map[string]string{"app": "vllm"}) @@ -337,6 +357,47 @@ func TestHandle_TypedVLLMWithoutCapabilityAnnotationsUsesDedicatedMPAdapter(t *t } } +func TestHandle_TypedNodeLocalVLLMGatesOnOwnershipVerifiedSameNodeServer(t *testing.T) { + const ns = "engines" + cb := typedVLLMNodeLocalBackend("node-cache", ns, map[string]string{"app": "vllm-node"}) + h := newHandler(t, cb) + pod := vllmEnginePod("engine-node", map[string]string{"app": "vllm-node"}) + pod.Spec.NodeSelector = map[string]string{"inference-system.io/pool": "owned"} + pod.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{{Weight: 1}}}} + wantNodeSelector := map[string]string{"inference-system.io/pool": "owned"} + wantAffinity := pod.Spec.Affinity.DeepCopy() + req := newRequest(t, pod, ns) + + resp := h.Handle(context.Background(), req) + if !resp.Allowed || len(resp.Patches) == 0 { + t.Fatalf("NodeLocal injection: Allowed=%v patches=%d result=%+v", resp.Allowed, len(resp.Patches), resp.Result) + } + mutated := applyPatches(t, req.Object.Raw, resp) + if findInitContainerByName(mutated.Spec.InitContainers, "lmcache-mp-server") != nil { + t.Fatalf("NodeLocal engine received PodLocal server: %+v", mutated.Spec.InitContainers) + } + gate := findInitContainerByName(mutated.Spec.InitContainers, "lmcache-node-local-gate") + if gate == nil || !strings.Contains(strings.Join(gate.Args, " "), "/healthcheck") || !strings.Contains(strings.Join(gate.Args, " "), "/config") { + t.Fatalf("ownership-verifying NodeLocal gate = %+v", gate) + } + config := testArgValue(mutated.Spec.Containers[0].Args, "--kv-transfer-config") + if !strings.Contains(config, `tcp://$(INFERENCECACHE_NODE_IP)`) { + t.Fatalf("node-derived vLLM config = %q", config) + } + if mutated.Spec.HostNetwork || mutated.Spec.HostIPC { + t.Fatalf("engine entered host namespace: hostNetwork=%v hostIPC=%v", mutated.Spec.HostNetwork, mutated.Spec.HostIPC) + } + if !reflect.DeepEqual(mutated.Spec.NodeSelector, wantNodeSelector) || !reflect.DeepEqual(mutated.Spec.Affinity, wantAffinity) { + t.Fatalf("NodeLocal mutated inference-owned placement: selector=%v affinity=%+v", mutated.Spec.NodeSelector, mutated.Spec.Affinity) + } + if got := mutated.Annotations[AnnotationInjectedByUID]; got != string(cb.UID) { + t.Fatalf("injected backend UID = %q", got) + } + if got := mutated.Labels[LabelLMCacheMPMetrics]; got != "" { + t.Fatalf("NodeLocal engine must not advertise a PodLocal metrics sidecar: %s=%q", LabelLMCacheMPMetrics, got) + } +} + func TestHandle_TypedPodLocalVLLMUsesDedicatedMPAdapter(t *testing.T) { const ns = "engines" cb := typedVLLMPodLocalBackend("vllm-typed", ns, map[string]string{"app": "vllm-mp"}) @@ -1422,61 +1483,22 @@ func TestHandle_EmptyEngineSelector_Skipped(t *testing.T) { } } -// TestHandle_OverlappingSelectors_FirstNameWins exercises the shared -// attribution rule between the pod webhook and the CacheIndex poller: -// when two CacheBackends in the same namespace have overlapping -// EngineSelectors that both match the engine pod, BOTH surfaces must -// pick the same backend — the one sorted first by metadata.name — -// otherwise the engine is wired to one backend's endpoint while -// status.indexParticipation reports the other as the owner. -// -// The matching poller-side assertion lives in -// TestRefreshOverlappingSelectorsFirstNameWins -// (internal/controller/cacheindex_controller_test.go). -func TestHandle_OverlappingSelectors_FirstNameWins(t *testing.T) { +func TestHandle_OverlappingSelectors_Rejected(t *testing.T) { const ns = "engines" - // Create the backends in non-alphabetical order so a name-sort is - // observably different from raw List order. cbZebra := readyCacheBackend("zebra", ns, map[string]string{"app": "vllm"}) - cbAlpha := readyCacheBackend("alpha", ns, map[string]string{"app": "vllm"}) + cbAlpha := readyCacheBackend("alpha", ns, map[string]string{"app": "vllm", "model": "qwen"}) h := newHandler(t, cbZebra, cbAlpha) - pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm"}) + pod := vllmEnginePod("engine-a", map[string]string{"app": "vllm", "model": "qwen"}) req := newRequest(t, pod, ns) resp := h.Handle(context.Background(), req) - if !resp.Allowed { - t.Fatalf("expected Allowed, got: %+v", resp.Result) + if resp.Allowed { + t.Fatalf("expected overlapping selectors to deny Pod admission, got Allowed with patches %+v", resp.Patches) } - if len(resp.Patches) == 0 { - t.Fatal("expected injection patches when at least one backend matches") - } - // The `inferencecache.io/injected-by` annotation records who claimed - // the pod. Sorted-by-name "alpha" must win over "zebra". - want := ns + "/alpha" - var got string - for _, p := range resp.Patches { - if p.Path == "/metadata/annotations" || p.Path == "/metadata/annotations/"+jsonPatchEscape(AnnotationInjectedBy) { - if anno, ok := p.Value.(map[string]any); ok { - if v, ok := anno[AnnotationInjectedBy].(string); ok { - got = v - } - } else if s, ok := p.Value.(string); ok { - got = s - } - } + if resp.Result == nil || !strings.Contains(resp.Result.Message, "multiple CacheBackends") || + !strings.Contains(resp.Result.Message, "alpha, zebra") { + t.Fatalf("denial message = %+v, want deterministic conflicting backend names", resp.Result) } - if got != want { - t.Fatalf("injected-by annotation = %q, want %q (deterministic name-sort: alpha < zebra)", got, want) - } -} - -// jsonPatchEscape is the JSON-Pointer escaping for "/" and "~" in JSON -// Patch paths (RFC 6901). Used here only to match the annotation path -// regardless of how the controller-runtime patch emitter renders it. -func jsonPatchEscape(s string) string { - s = strings.ReplaceAll(s, "~", "~0") - s = strings.ReplaceAll(s, "/", "~1") - return s } func TestHandle_ListError_FailOpen(t *testing.T) { diff --git a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go index 63a614e4..c06f9e8d 100644 --- a/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go +++ b/internal/webhook/v1alpha1/cachebackend_defaulter_envtest_test.go @@ -11,6 +11,8 @@ import ( "testing" "time" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -127,6 +129,7 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) mvCR := validPodLocalMPBackend() mvCR.Name = "minimum" mvCR.Namespace = "team-a" + mvCR.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "minimum"} mvCR.Spec.Type = "" mvCR.Spec.Integration = nil mvCR.Spec.Observation = nil @@ -178,6 +181,7 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) canonicalCR := validPodLocalMPBackend() canonicalCR.Name = "canonical-host-only" canonicalCR.Namespace = "team-a" + canonicalCR.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "canonical"} canonicalCR.Spec.Runtime = cachev1alpha1.CacheBackendRuntimeSGLang canonicalCR.Spec.RemoteStorage = nil if err := k8s.Create(ctx, canonicalCR); err != nil { @@ -209,7 +213,7 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) Type: cachev1alpha1.CacheBackendTypeSGLangHiCache, HiCache: &cachev1alpha1.SGLangHiCacheSpec{Ratio: "2"}, EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{ - MatchLabels: map[string]string{"app.kubernetes.io/name": "sglang"}, + MatchLabels: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "explicit"}, }, Integration: &cachev1alpha1.CacheBackendIntegrationSpec{ Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, @@ -230,12 +234,12 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) // --- Current MP API CREATE/UPDATE compatibility --- // - // The real apiserver must accept the new PodLocal shape, preserve it on an - // unrelated update, and reject an update that selects the reserved NodeLocal - // topology before it is implemented. + // The real apiserver must accept both typed MP topologies and enforce the + // NodeLocal host/scheduling contract on CREATE and UPDATE. mpCR := validPodLocalMPBackend() mpCR.Name = "podlocal-mp" mpCR.Namespace = "team-a" + mpCR.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "podlocal"} if err := k8s.Create(ctx, mpCR); err != nil { t.Fatalf("PodLocal MP CacheBackend should be admitted: %v", err) } @@ -254,10 +258,63 @@ func TestCacheBackendDefaulter_MinimumViableYAMLGetsFullyDefaulted(t *testing.T) if err := k8s.Update(ctx, &persistedMP); err != nil { t.Fatalf("unrelated update on PodLocal MP object should be admitted: %v", err) } - persistedMP.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal - persistedMP.Spec.LMCache.PodLocal = nil - persistedMP.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} - if err := k8s.Update(ctx, &persistedMP); err == nil { - t.Fatal("update selecting unimplemented NodeLocal topology should be rejected") + toNodeLocal := func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, HTTPPort: 18080, L1Capacity: resource.MustParse("4Gi"), MaxGPUWorkers: 4, MaxCPUWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + } + } + toNodeLocal(&persistedMP) + if err := k8s.Update(ctx, &persistedMP); err != nil { + t.Fatalf("valid PodLocal-to-NodeLocal update should be admitted: %v", err) + } + + nodeLocalCreate := validPodLocalMPBackend() + nodeLocalCreate.Name = "nodelocal-create" + nodeLocalCreate.Namespace = "team-a" + nodeLocalCreate.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "nodelocal"} + toNodeLocal(nodeLocalCreate) + if err := k8s.Create(ctx, nodeLocalCreate); err != nil { + t.Fatalf("valid NodeLocal CREATE should be admitted: %v", err) + } + var persistedNodeLocal cachev1alpha1.CacheBackend + if err := live.Get(ctx, client.ObjectKeyFromObject(nodeLocalCreate), &persistedNodeLocal); err != nil { + t.Fatalf("get back NodeLocal CR: %v", err) + } + persistedNodeLocal.Spec.LMCache.NodeLocal.Server.HTTPPort = persistedNodeLocal.Spec.LMCache.NodeLocal.Server.Port + if err := k8s.Update(ctx, &persistedNodeLocal); err == nil { + t.Fatal("NodeLocal UPDATE with colliding host ports should be rejected") + } + + // A second CacheBackend cannot own the same namespace-scoped cache domain. + overlap := validPodLocalMPBackend() + overlap.Name = "overlapping-selector" + overlap.Namespace = "team-a" + overlap.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "nodelocal"} + if err := k8s.Create(ctx, overlap); err == nil { + t.Fatal("CacheBackend CREATE with overlapping engineSelector should be rejected") + } + + // Ownership is intentionally independent of ordinary Pod labels: the + // canonical selector contains only cache-domain, even when those Pods carry + // app/model/environment labels for other Kubernetes consumers. + extraSelector := validPodLocalMPBackend() + extraSelector.Name = "extra-selector-label" + extraSelector.Namespace = "team-a" + extraSelector.Spec.EngineSelector.MatchLabels = map[string]string{ + cachev1alpha1.CacheBackendDomainLabel: "extra-selector-label", + "app": "vllm", + } + if err := k8s.Create(ctx, extraSelector); err == nil { + t.Fatal("CacheBackend CREATE with a second ownership selector label should be rejected") } } diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go index 4d65e667..feaf990d 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation.go @@ -72,13 +72,19 @@ func validateLMCacheTopology(cb *cachev1alpha1.CacheBackend) field.ErrorList { "required when topology=NodeLocal")) } else { errs = append(errs, validateNodeLocalServer(lm.NodeLocal.Server, lmPath.Child("nodeLocal", "server"))...) + if lm.NodeLocal.IdleRetentionSeconds < 0 || lm.NodeLocal.IdleRetentionSeconds > 86400 { + errs = append(errs, field.Invalid(lmPath.Child("nodeLocal", "idleRetentionSeconds"), lm.NodeLocal.IdleRetentionSeconds, + "must be between 0 and 86400 seconds")) + } } if lm.PodLocal != nil { errs = append(errs, field.Forbidden(lmPath.Child("podLocal"), "must be omitted when topology=NodeLocal")) } - errs = append(errs, field.Forbidden(lmPath.Child("topology"), - "NodeLocal is reserved for Phase 8 and is not implemented; use PodLocal")) + if cb.Spec.EngineSelector == nil || len(cb.Spec.EngineSelector.MatchLabels) == 0 { + errs = append(errs, field.Required(field.NewPath("spec", "engineSelector", "matchLabels"), + "NodeLocal requires a non-empty engine selector to define its trust domain")) + } default: errs = append(errs, field.NotSupported(lmPath.Child("topology"), lm.Topology, []string{string(cachev1alpha1.LMCacheTopologyPodLocal), string(cachev1alpha1.LMCacheTopologyNodeLocal)})) @@ -97,6 +103,10 @@ func validatePodLocalServer(server *cachev1alpha1.LMCachePodLocalServerSpec, pat &server.L1Capacity, server.Resources, path, + map[int32]string{ + lmcacheKVEventPort: "engine KV-event publisher", + lmcacheMPHTTPPort: "LMCache MP HTTP health/control", + }, ) if server.MaxWorkers < 1 { errs = append(errs, field.Invalid(path.Child("maxWorkers"), server.MaxWorkers, "must be at least 1")) @@ -114,6 +124,7 @@ func validateNodeLocalServer(server *cachev1alpha1.LMCacheNodeLocalServerSpec, p &server.L1Capacity, server.Resources, path, + nil, ) if server.MaxGPUWorkers < 1 { errs = append(errs, field.Invalid(path.Child("maxGPUWorkers"), server.MaxGPUWorkers, "must be at least 1")) @@ -121,6 +132,34 @@ func validateNodeLocalServer(server *cachev1alpha1.LMCacheNodeLocalServerSpec, p if server.MaxCPUWorkers < 1 { errs = append(errs, field.Invalid(path.Child("maxCPUWorkers"), server.MaxCPUWorkers, "must be at least 1")) } + ports := []struct { + name string + port int32 + }{ + {name: "port", port: server.Port}, + {name: "httpPort", port: server.HTTPPort}, + } + seen := map[int32]string{} + for _, item := range ports { + if item.port < 1 || item.port > 65535 { + errs = append(errs, field.Invalid(path.Child(item.name), item.port, "must be between 1 and 65535")) + continue + } + if prior, ok := seen[item.port]; ok { + errs = append(errs, field.Invalid(path.Child(item.name), item.port, + fmt.Sprintf("must be distinct from %s because both listeners bind the node network namespace", prior))) + continue + } + seen[item.port] = item.name + } + if _, requested := server.Resources.Requests[corev1.ResourceName("nvidia.com/gpu")]; requested { + errs = append(errs, field.Forbidden(path.Child("resources", "requests").Key("nvidia.com/gpu"), + "NodeLocal MP servers use NVIDIA_VISIBLE_DEVICES=all for CUDA IPC and must not reserve engine allocatable GPUs")) + } + if _, limited := server.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")]; limited { + errs = append(errs, field.Forbidden(path.Child("resources", "limits").Key("nvidia.com/gpu"), + "NodeLocal MP servers use NVIDIA_VISIBLE_DEVICES=all for CUDA IPC and must not reserve engine allocatable GPUs")) + } return errs } @@ -130,6 +169,7 @@ func validateMPServer( l1Capacity *resource.Quantity, resources corev1.ResourceRequirements, path *field.Path, + reservedPorts map[int32]string, ) field.ErrorList { var errs field.ErrorList trimmedImage := strings.TrimSpace(image) @@ -143,12 +183,9 @@ func validateMPServer( if port < 1 || port > 65535 { errs = append(errs, field.Invalid(path.Child("port"), port, "must be between 1 and 65535")) - } else if port == lmcacheKVEventPort { - errs = append(errs, field.Invalid(path.Child("port"), port, - fmt.Sprintf("collides with the engine KV-event publisher port %d", lmcacheKVEventPort))) - } else if port == lmcacheMPHTTPPort { + } else if owner, reserved := reservedPorts[port]; reserved { errs = append(errs, field.Invalid(path.Child("port"), port, - fmt.Sprintf("collides with the LMCache MP HTTP health/control port %d", lmcacheMPHTTPPort))) + fmt.Sprintf("collides with the %s port %d", owner, port))) } if l1Capacity == nil || l1Capacity.Sign() <= 0 { @@ -261,7 +298,7 @@ func validateMPServerResourceRequirements(resources corev1.ResourceRequirements, } // rejectUnimplementedRedisBindingFeatures permits authentication for typed -// PodLocal LMCache MP adapters while keeping every unsupported LMCache 0.5.3 +// LMCache MP adapters while keeping every unsupported LMCache 0.5.3 // RESP feature explicit. The common MP server renderer supports // username/password for both SGLang and vLLM, but not TLS or logical database // selection. Managed Redis currently provisions the default user, so its @@ -275,14 +312,15 @@ func rejectUnimplementedRedisBindingFeatures(cb *cachev1alpha1.CacheBackend) fie var errs field.ErrorList if redis.Authentication != nil { authPath := path.Child("authentication") - isTypedPodLocalMP := cb.Spec.LMCache != nil && + isTypedMP := cb.Spec.LMCache != nil && cb.Spec.EffectiveCacheType() == cachev1alpha1.CacheBackendTypeLMCache && - cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal && + (cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyPodLocal || + cb.Spec.LMCache.Topology == cachev1alpha1.LMCacheTopologyNodeLocal) && (cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeSGLang || cb.Spec.Runtime == cachev1alpha1.CacheBackendRuntimeVLLM) - if !isTypedPodLocalMP { + if !isTypedMP { errs = append(errs, field.Forbidden(authPath, - "Redis authentication is currently rendered only by a typed PodLocal LMCache MP adapter")) + "Redis authentication is currently rendered only by a typed LMCache MP adapter")) } else { if redis.Authentication.Username != nil { errs = append(errs, validateRedisSecretKeySelector(*redis.Authentication.Username, authPath.Child("username"))...) diff --git a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go index 42863281..e77ffcfd 100644 --- a/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go +++ b/internal/webhook/v1alpha1/cachebackend_lmcache_mp_validation_test.go @@ -36,7 +36,7 @@ func validPodLocalMPBackend() *cachev1alpha1.CacheBackend { }}, }, Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "engine"}}, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "engine"}}, }, } } @@ -160,7 +160,7 @@ func TestValidateRedisAuthenticationForTypedPodLocal(t *testing.T) { cb.Spec.LMCache.Topology = "" cb.Spec.LMCache.PodLocal = nil errs := rejectUnimplementedRedisBindingFeatures(cb) - if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed PodLocal LMCache MP") { + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed LMCache MP") { t.Fatalf("errors = %v, want topology-scoped rejection", errs) } }) @@ -169,7 +169,7 @@ func TestValidateRedisAuthenticationForTypedPodLocal(t *testing.T) { cb := newAuthBackend(cachev1alpha1.CacheBackendRemoteStorageOwnershipExternal) cb.Spec.Type = cachev1alpha1.CacheBackendTypeSGLangHiCache errs := rejectUnimplementedRedisBindingFeatures(cb) - if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed PodLocal LMCache MP") { + if len(errs) != 1 || !strings.Contains(errs[0].Error(), "typed LMCache MP") { t.Fatalf("errors = %v, want cache-type-scoped rejection", errs) } }) @@ -187,13 +187,70 @@ func TestValidateLMCacheTopologyRequiresTypedPodLocal(t *testing.T) { } } -func TestValidateLMCacheTopologyRejectsNodeLocalUntilImplemented(t *testing.T) { +func TestValidateLMCacheTopologyAcceptsCompleteNodeLocal(t *testing.T) { cb := validPodLocalMPBackend() cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal cb.Spec.LMCache.PodLocal = nil - cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} - if errs := validateLMCacheTopology(cb); len(errs) == 0 { - t.Fatal("NodeLocal was accepted before Phase 8") + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, HTTPPort: 18080, + L1Capacity: resource.MustParse("4Gi"), MaxGPUWorkers: 4, MaxCPUWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + } + if errs := validateLMCacheTopology(cb); len(errs) != 0 { + t.Fatalf("complete NodeLocal errors: %v", errs) + } +} + +func TestValidateLMCacheNodeLocalSafetyContract(t *testing.T) { + base := func() *cachev1alpha1.CacheBackend { + cb := validPodLocalMPBackend() + cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cb.Spec.LMCache.PodLocal = nil + cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{ + Server: &cachev1alpha1.LMCacheNodeLocalServerSpec{ + Image: "registry.example/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6555, HTTPPort: 18080, + L1Capacity: resource.MustParse("4Gi"), MaxGPUWorkers: 4, MaxCPUWorkers: 4, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("5Gi")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("5Gi")}, + }, + }, + Scheduling: &cachev1alpha1.LMCacheNodeLocalSchedulingSpec{}, + } + return cb + } + tests := []struct { + name string + mutate func(*cachev1alpha1.CacheBackend) + wantField string + }{ + {name: "distinct HTTP host port", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.NodeLocal.Server.HTTPPort = 6555 }, wantField: "httpPort"}, + {name: "no GPU reservation", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.NodeLocal.Server.Resources.Requests[corev1.ResourceName("nvidia.com/gpu")] = resource.MustParse("1") + cb.Spec.LMCache.NodeLocal.Server.Resources.Limits[corev1.ResourceName("nvidia.com/gpu")] = resource.MustParse("1") + }, wantField: "nvidia.com/gpu"}, + {name: "required trust-domain selector", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.EngineSelector = nil }, wantField: "engineSelector"}, + {name: "bounded idle retention", mutate: func(cb *cachev1alpha1.CacheBackend) { + cb.Spec.LMCache.NodeLocal.IdleRetentionSeconds = 86401 + }, wantField: "idleRetentionSeconds"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := base() + tc.mutate(cb) + errs := validateLMCacheTopology(cb) + if len(errs) == 0 || !strings.Contains(errs.ToAggregate().Error(), tc.wantField) { + t.Fatalf("errors = %v, want field %q", errs, tc.wantField) + } + }) } } @@ -230,11 +287,11 @@ func TestValidateLMCacheTopologyCurrentMatrix(t *testing.T) { {name: "mixed NodeLocal block", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} }, wantField: "spec.lmCache.nodeLocal"}, - {name: "NodeLocal reserved", mutate: func(cb *cachev1alpha1.CacheBackend) { + {name: "NodeLocal missing server", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal cb.Spec.LMCache.PodLocal = nil cb.Spec.LMCache.NodeLocal = &cachev1alpha1.LMCacheNodeLocalSpec{} - }, wantField: "spec.lmCache.topology"}, + }, wantField: "spec.lmCache.nodeLocal.server"}, {name: "digest without repository", mutate: func(cb *cachev1alpha1.CacheBackend) { cb.Spec.LMCache.PodLocal.Server.Image = "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, wantField: "spec.lmCache.podLocal.server.image"}, diff --git a/internal/webhook/v1alpha1/cachebackend_validator.go b/internal/webhook/v1alpha1/cachebackend_validator.go index adafd9aa..d13581d7 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator.go +++ b/internal/webhook/v1alpha1/cachebackend_validator.go @@ -7,12 +7,16 @@ package v1alpha1 import ( "context" "fmt" + "sort" + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" adapterruntime "github.com/cachebox-project/inference-cache/pkg/adapters/runtime" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime/schema" + kvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" ) @@ -35,6 +39,11 @@ type CacheBackendValidator struct { // Registry resolves the runtime adapter for a (runtime, backend) pair at // admission time. The composition root must inject it. Registry *adapterruntime.Registry + + // Reader performs namespace-scoped cross-object checks that cannot be + // represented in the CRD schema. Production wiring uses the manager's live + // APIReader so a newly-created sibling is visible at admission time. + Reader client.Reader } // ValidationRule is the seam plugged-in admission rules implement. It @@ -49,6 +58,7 @@ type ValidationRule func(cb *cachev1alpha1.CacheBackend) field.ErrorList // [CacheBackendValidator.Rules]) to extend admission; no other code in the // handler changes. var DefaultValidationRules = []ValidationRule{ + validateEngineSelectorDomain, validateCacheHierarchy, validateLMCacheTopology, rejectUnimplementedRedisBindingFeatures, @@ -82,7 +92,7 @@ func SetupCacheBackendWebhookWithManager(mgr ctrl.Manager, registry *adapterrunt } return ctrl.NewWebhookManagedBy(mgr, &cachev1alpha1.CacheBackend{}). WithDefaulter(&CacheBackendDefaulter{}). - WithValidator(&CacheBackendValidator{Registry: registry}). + WithValidator(&CacheBackendValidator{Registry: registry, Reader: mgr.GetAPIReader()}). Complete() } @@ -94,7 +104,14 @@ func SetupCacheBackendWebhookWithManager(mgr ctrl.Manager, registry *adapterrunt func (v *CacheBackendValidator) ValidateCreate(ctx context.Context, cb *cachev1alpha1.CacheBackend) (admission.Warnings, error) { logf.FromContext(ctx).V(1).Info("validating CacheBackend create", "namespace", cb.Namespace, "name", cb.Name, "type", cb.Spec.Type) - return collectWarnings(cb), v.validate(cb) + warnings := collectWarnings(cb) + errs := v.collectErrors(cb) + selectorErrs, err := v.checkOverlappingEngineSelectors(ctx, cb) + if err != nil { + return warnings, err + } + errs = append(errs, selectorErrs...) + return warnings, invalidCacheBackend(cb, errs) } // collectWarnings returns non-blocking advisories surfaced to the operator at @@ -115,19 +132,31 @@ func collectWarnings(cb *cachev1alpha1.CacheBackend) admission.Warnings { // // This is the standard pattern for tightening admission rules on a // v1alpha1 CRD: create-time is strict; update-time only rejects fresh -// violations so existing CRs aren't trapped. Without it, adding a new -// field-level rule would break every existing CR that happens to violate it -// the moment an operator runs `kubectl annotate` on it. +// single-object violations so existing CRs aren't trapped. The canonical +// cache-domain selector contract and selector overlap are exceptions: both are +// enforced on every UPDATE because they define live engine ownership. func (v *CacheBackendValidator) ValidateUpdate(ctx context.Context, oldCB, newCB *cachev1alpha1.CacheBackend) (admission.Warnings, error) { logf.FromContext(ctx).V(1).Info("validating CacheBackend update", "namespace", newCB.Namespace, "name", newCB.Name, "type", newCB.Spec.Type) warnings := collectWarnings(newCB) newErrs := v.collectErrors(newCB) - if len(newErrs) == 0 { + newSelectorErrs, err := v.checkOverlappingEngineSelectors(ctx, newCB) + if err != nil { + return warnings, err + } + if len(newErrs) == 0 && len(newSelectorErrs) == 0 { return warnings, nil } oldErrs := v.collectErrors(oldCB) introduced := filterIntroducedErrors(oldErrs, newErrs) + // Unlike ordinary validation tightening, engine ownership is never + // grandfathered: every updated object must use the canonical domain selector. + introduced = append(introduced, + filterIntroducedErrors(introduced, validateEngineSelectorDomain(newCB))...) + // Never grandfather a cross-object ownership conflict. This forces a + // concurrent-CREATE conflict to be corrected before any spec/metadata update + // can proceed, while DELETE remains unconditionally allowed by ValidateDelete. + introduced = append(introduced, newSelectorErrs...) if len(introduced) == 0 { return warnings, nil } @@ -138,6 +167,30 @@ func (v *CacheBackendValidator) ValidateUpdate(ctx context.Context, oldCB, newCB ) } +// validateEngineSelectorDomain keeps cache ownership independent of mutable +// application, runtime, model, and environment labels. A CacheBackend either +// has no selector or selects exactly one namespace-scoped compatibility-domain +// value. +func validateEngineSelectorDomain(cb *cachev1alpha1.CacheBackend) field.ErrorList { + if cb.Spec.EngineSelector == nil || len(cb.Spec.EngineSelector.MatchLabels) == 0 { + return nil + } + path := field.NewPath("spec", "engineSelector", "matchLabels") + domain, found := cb.Spec.EngineSelector.MatchLabels[cachev1alpha1.CacheBackendDomainLabel] + if !found || domain == "" { + return field.ErrorList{field.Required(path.Key(cachev1alpha1.CacheBackendDomainLabel), + "every non-empty engineSelector must declare one cache compatibility domain")} + } + if problems := kvalidation.IsValidLabelValue(domain); len(problems) > 0 { + return field.ErrorList{field.Invalid(path.Key(cachev1alpha1.CacheBackendDomainLabel), domain, problems[0])} + } + if len(cb.Spec.EngineSelector.MatchLabels) != 1 { + return field.ErrorList{field.Forbidden(path, + fmt.Sprintf("must contain only %q; engine Pods may carry other labels, but they do not define CacheBackend ownership", cachev1alpha1.CacheBackendDomainLabel))} + } + return nil +} + // ValidateDelete implements [admission.Validator]. Deletion is always // allowed: removing a CacheBackend that was previously admitted under a // stricter rule must still succeed so operators can clear bad state. @@ -151,7 +204,10 @@ func (v *CacheBackendValidator) ValidateDelete(_ context.Context, _ *cachev1alph // collectErrors directly so it can diff old vs new and only reject // newly introduced violations. func (v *CacheBackendValidator) validate(cb *cachev1alpha1.CacheBackend) error { - errs := v.collectErrors(cb) + return invalidCacheBackend(cb, v.collectErrors(cb)) +} + +func invalidCacheBackend(cb *cachev1alpha1.CacheBackend, errs field.ErrorList) error { if len(errs) == 0 { return nil } @@ -162,6 +218,40 @@ func (v *CacheBackendValidator) validate(cb *cachev1alpha1.CacheBackend) error { ) } +// checkOverlappingEngineSelectors rejects duplicate cache-domain ownership in +// one namespace. +// Like every list-then-admit cross-object rule, this is best effort under two +// exactly concurrent CREATEs. The Pod webhook is the authoritative runtime +// backstop: it rejects admission when racing objects produce more than one +// match instead of choosing a backend by name. +func (v *CacheBackendValidator) checkOverlappingEngineSelectors(ctx context.Context, cb *cachev1alpha1.CacheBackend) (field.ErrorList, error) { + selector := cb.Spec.EngineSelector + if selector == nil || len(selector.MatchLabels) == 0 || v.Reader == nil { + return nil, nil + } + var siblings cachev1alpha1.CacheBackendList + if err := v.Reader.List(ctx, &siblings, client.InNamespace(cb.Namespace)); err != nil { + return nil, fmt.Errorf("listing CacheBackends in namespace %q for engineSelector overlap: %w", cb.Namespace, err) + } + sort.Slice(siblings.Items, func(i, j int) bool { return siblings.Items[i].Name < siblings.Items[j].Name }) + for i := range siblings.Items { + other := &siblings.Items[i] + if other.Name == cb.Name || other.Spec.EngineSelector == nil || + len(other.Spec.EngineSelector.MatchLabels) == 0 { + continue + } + domain := selector.MatchLabels[cachev1alpha1.CacheBackendDomainLabel] + otherDomain := other.Spec.EngineSelector.MatchLabels[cachev1alpha1.CacheBackendDomainLabel] + if domain != "" && domain == otherDomain { + return field.ErrorList{field.Forbidden( + field.NewPath("spec", "engineSelector", "matchLabels").Key(cachev1alpha1.CacheBackendDomainLabel), + fmt.Sprintf("cache domain %q is already owned by CacheBackend %q in namespace %q", domain, other.Name, cb.Namespace), + )}, nil + } + } + return nil, nil +} + // collectErrors returns the field-scoped violations every configured // rule produced for cb, including the runtime-adapter compatibility // check. Centralised so ValidateCreate and ValidateUpdate share the diff --git a/internal/webhook/v1alpha1/cachebackend_validator_test.go b/internal/webhook/v1alpha1/cachebackend_validator_test.go index bb9e653d..343f8a17 100644 --- a/internal/webhook/v1alpha1/cachebackend_validator_test.go +++ b/internal/webhook/v1alpha1/cachebackend_validator_test.go @@ -12,7 +12,9 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/client/fake" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" builtinruntime "github.com/cachebox-project/inference-cache/internal/adapters/builtin/runtime" @@ -83,6 +85,136 @@ func TestSetupCacheBackendWebhookRequiresRegistry(t *testing.T) { } } +func TestValidatorRejectsOverlappingEngineSelectorsInNamespace(t *testing.T) { + existing := newBackend() + existing.Name = "broad" + existing.Namespace = "team-a" + existing.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + + tests := []struct { + name string + selector map[string]string + wantErr bool + }{ + {name: "same domain", selector: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"}, wantErr: true}, + {name: "different domain is disjoint", selector: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "sglang"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + candidate := newBackend() + candidate.Name = "candidate" + candidate.Namespace = "team-a" + candidate.Spec.EngineSelector.MatchLabels = tc.selector + _, err := selectorValidator(t, existing).ValidateCreate(context.Background(), candidate) + if tc.wantErr { + if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "already owned") { + t.Fatalf("ValidateCreate error = %v, want overlapping-selector Invalid", err) + } + return + } + if err != nil { + t.Fatalf("ValidateCreate rejected disjoint selector: %v", err) + } + }) + } +} + +func TestValidatorEngineSelectorOverlapIsNamespaceScoped(t *testing.T) { + existing := newBackend() + existing.Name = "other-namespace" + existing.Namespace = "team-b" + existing.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + candidate := newBackend() + candidate.Name = "candidate" + candidate.Namespace = "team-a" + candidate.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + if _, err := selectorValidator(t, existing).ValidateCreate(context.Background(), candidate); err != nil { + t.Fatalf("same selector in another namespace rejected: %v", err) + } +} + +func TestValidatorRejectsUpdateThatIntroducesSelectorOverlap(t *testing.T) { + existing := newBackend() + existing.Name = "existing" + existing.Namespace = "team-a" + existing.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + oldCB := newBackend() + oldCB.Name = "candidate" + oldCB.Namespace = "team-a" + oldCB.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "sglang"} + newCB := oldCB.DeepCopy() + newCB.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + + _, err := selectorValidator(t, existing).ValidateUpdate(context.Background(), oldCB, newCB) + if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "already owned") { + t.Fatalf("ValidateUpdate error = %v, want newly-overlapping selector Invalid", err) + } +} + +func TestValidatorRejectsUnrelatedUpdateWhileSelectorOverlapExists(t *testing.T) { + existing := newBackend() + existing.Name = "existing" + existing.Namespace = "team-a" + existing.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + oldCB := newBackend() + oldCB.Name = "candidate" + oldCB.Namespace = "team-a" + oldCB.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"} + newCB := oldCB.DeepCopy() + newCB.Labels = map[string]string{"maintenance": "requested"} + + _, err := selectorValidator(t, existing).ValidateUpdate(context.Background(), oldCB, newCB) + if err == nil || !apierrors.IsInvalid(err) || !strings.Contains(err.Error(), "already owned") { + t.Fatalf("unrelated update error = %v, want existing overlap to remain blocked", err) + } +} + +func TestValidatorRequiresSoleCacheDomainSelector(t *testing.T) { + tests := []struct { + name string + selector map[string]string + want string + }{ + {name: "app selector is not an ownership domain", selector: map[string]string{"app": "vllm"}, want: cachev1alpha1.CacheBackendDomainLabel}, + {name: "extra ownership key", selector: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm", "app": "vllm"}, want: "must contain only"}, + {name: "invalid domain value", selector: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "Qwen/Qwen"}, want: "Invalid value"}, + {name: "canonical domain", selector: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "vllm"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := newBackend() + cb.Spec.EngineSelector.MatchLabels = tc.selector + _, err := shippingValidator().ValidateCreate(context.Background(), cb) + if tc.want == "" { + if err != nil { + t.Fatalf("canonical selector rejected: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("ValidateCreate error = %v, want %q", err, tc.want) + } + }) + } +} + +func TestValidatorRejectsNonCanonicalSelectorOnEveryUpdate(t *testing.T) { + oldCB := newBackend() + oldCB.Spec.EngineSelector.MatchLabels = map[string]string{"app": "legacy"} + + unrelated := oldCB.DeepCopy() + unrelated.Labels = map[string]string{"maintenance": "requested"} + if _, err := shippingValidator().ValidateUpdate(context.Background(), oldCB, unrelated); err == nil || !strings.Contains(err.Error(), cachev1alpha1.CacheBackendDomainLabel) { + t.Fatalf("non-canonical selector update error = %v, want strict domain requirement", err) + } + + canonical := oldCB.DeepCopy() + canonical.Spec.EngineSelector.MatchLabels = map[string]string{cachev1alpha1.CacheBackendDomainLabel: "canonical"} + if _, err := shippingValidator().ValidateUpdate(context.Background(), oldCB, canonical); err != nil { + t.Fatalf("canonical selector update rejected: %v", err) + } +} + func TestValidator_VLLMRoleReadOnlyRejected(t *testing.T) { // vLLM renders ReadOnly as kv_consumer, but LMCache 0.5.3 does not enforce // that directionality. Admission must reject the unsupported API promise. @@ -106,6 +238,18 @@ func shippingValidator() *CacheBackendValidator { return &CacheBackendValidator{Registry: defaultShippingRegistry()} } +func selectorValidator(t *testing.T, existing ...*cachev1alpha1.CacheBackend) *CacheBackendValidator { + t.Helper() + objects := make([]runtime.Object, len(existing)) + for i := range existing { + objects[i] = existing[i] + } + return &CacheBackendValidator{ + Registry: defaultShippingRegistry(), + Reader: fake.NewClientBuilder().WithScheme(newCacheScheme(t)).WithRuntimeObjects(objects...).Build(), + } +} + type rejectingBindingAdapter struct { adapterruntime.KVCacheRuntimeAdapter } @@ -134,7 +278,7 @@ func newHiCacheBackend() *cachev1alpha1.CacheBackend { Mode: cachev1alpha1.CacheBackendIntegrationModeOffload, Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, }, - EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{"app": "sglang"}}, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{cachev1alpha1.CacheBackendDomainLabel: "sglang"}}, }, } } diff --git a/internal/webhook/v1alpha1/doc.go b/internal/webhook/v1alpha1/doc.go index 8110f4ee..4074276a 100644 --- a/internal/webhook/v1alpha1/doc.go +++ b/internal/webhook/v1alpha1/doc.go @@ -11,7 +11,8 @@ // and rejects configurations that can't be reconciled at all (External // backend without an Endpoint, a cross-namespace Endpoint that wasn't // explicitly opted into, an unsupported runtime/backend pair, reserved -// engineOverrides). +// engineOverrides, or an engine selector overlapping another backend in +// the namespace). // - CachePolicy enforces at most one policy per namespace (the reconciler // flattens to one ResolvedPolicy per namespace, so a second CR silently // loses) and a strictly positive evictionTTL when set. diff --git a/site/content/en/docs/reference/cli-doctor.md b/site/content/en/docs/reference/cli-doctor.md index 4c7dd90d..985c37e3 100644 --- a/site/content/en/docs/reference/cli-doctor.md +++ b/site/content/en/docs/reference/cli-doctor.md @@ -27,7 +27,7 @@ order. Finding codes are stable and greppable; severity is shown in parentheses. | 3 | `/policy` wired | `PL001` (FAIL), `PL002` (OK), `PL003` (WARN) | | 4 | `/probe` wired | `PB001` (FAIL), `PB002` (OK), `PB003` (WARN) | | 5 | Per-`CacheBackend` health | `CB001`–`CB005` (WARN), `CB006` (OK), `CB007` (WARN — `FunctionalProbeOK` not True) | -| 6 | Engine-pod injection audit | `EP001` (WARN), `EP002` (OK) | +| 6 | Engine-pod injection audit | `EP001`/`EP003` (WARN), `EP002` (OK) | | 7 | Orphan-pod check | `OP001` (WARN) | | 8 | `CacheTenant` health | `CT001` (WARN), `CT002` (OK) | | 9 | `CachePolicy` coverage | `CP001` (INFO), `CP002` (OK), `CP003` (WARN — >1 CachePolicy in a namespace) | From e6e86de9f596d488bf9deafff20305513b4fe0d4 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 12 Aug 2026 20:53:29 -0700 Subject: [PATCH 09/13] Fix PR CI and review blockers Signed-off-by: Yue Sun --- Makefile | 4 +- .../gpu-validation/kustomization.yaml | 20 +-- .../lmcache-multiprocess-migration-roadmap.md | 8 +- docs/design/lmcache-server-persistence.md | 34 ++--- docs/design/sglang-lmcache-mp-mode.md | 140 +++++------------- docs/reference-stack/GPU-RUNBOOK.md | 30 +--- docs/reference-stack/VERSIONS.md | 2 +- .../scripts/default_install_smoke.sh | 8 +- hack/verify-samples/admission_test.go | 19 +++ .../builtin/runtime/runtime_helpers.go | 9 +- .../cachebackend_kvevent_gate_test.go | 98 +++++++++++- ...chebackend_schema_trim_integration_test.go | 13 ++ 12 files changed, 202 insertions(+), 183 deletions(-) diff --git a/Makefile b/Makefile index 1a12b5d4..b0a548e8 100644 --- a/Makefile +++ b/Makefile @@ -538,8 +538,8 @@ install-hooks: ## Install git hooks (vendor-neutral naming guard) via core.hooks .PHONY: verify-naming verify-naming: ## Fail if core-identity files reference OCI/Oracle (see CONTRIBUTING.md). - @bad=$$(grep -rniEI '\boci\b|oci\.com|oraclecloud|\boracle\b' \ - api proto gen pkg config/crd config/rbac config/default config/manager config/observability config/samples config/server config/webhook config/certmanager config/overlays docs/observability internal PROJECT go.mod 2>/dev/null || true); \ + @bad=$$(grep -rniEI '(^|[^[:alnum:]_])(oci|oracle)([^[:alnum:]_]|$$)|ocir\.io|oci\.com|oraclecloud' \ + api proto gen pkg config/crd config/rbac config/default config/manager config/observability config/samples config/server config/webhook config/certmanager config/overlays docs/design docs/observability docs/reference-stack internal PROJECT go.mod 2>/dev/null || true); \ if [ -n "$$bad" ]; then \ echo "✗ OCI/Oracle reference in core-identity files (banned per CONTRIBUTING.md):"; \ echo "$$bad" | sed 's/^/ /'; \ diff --git a/config/overlays/gpu-validation/kustomization.yaml b/config/overlays/gpu-validation/kustomization.yaml index 969be60d..bcfda7e7 100644 --- a/config/overlays/gpu-validation/kustomization.yaml +++ b/config/overlays/gpu-validation/kustomization.yaml @@ -6,28 +6,12 @@ # cluster-wide for its CRD/controller surfaces, but ask the API server to send # Pod CREATE admission requests to the inference-cache mutator only from the # dedicated test namespace. CacheBackend/CachePolicy/CacheTenant admission is -# intentionally unchanged. +# intentionally unchanged. Image selection is environment-owned; set the +# controller, server, and optional subscriber images at deployment time. resources: - ../../default -images: -- name: ghcr.io/cachebox-project/inference-cache-controller - newName: sjc.ocir.io/idqj093njucb/inference-cache-controller - digest: sha256:6dcab2344027ef8ac3db2ab22352cdaa77d80202ec11df49dddeeefe08095b18 -- name: ghcr.io/cachebox-project/inference-cache-server - newName: sjc.ocir.io/idqj093njucb/inference-cache-server - digest: sha256:f735a5e69280411995f1e15d1a19b40c462e450bcf7ea0012a0c5520fb778d46 - patches: -- target: - group: apps - version: v1 - kind: Deployment - name: inference-cache-controller-manager - patch: | - - op: add - path: /spec/template/spec/containers/0/args/- - value: --kvevent-subscriber-image=sjc.ocir.io/idqj093njucb/inference-cache-subscriber@sha256:2fdaa611642a0f2c48b6c7a7257ff28d75030e4bf6df3cebb589319f2e48e504 - target: group: admissionregistration.k8s.io version: v1 diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 8c866e91..9d55c07c 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -651,7 +651,7 @@ Live validation ran on 2026-08-10/11 in SJC dev: | Item | Evidence | |---|---| | Environment | Kubernetes 1.31.1; A100-SXM4-80GB; driver 550.163.01; CUDA 12.9 | -| Engine | `us-sanjose-1.ocir.io/idqj093njucb/vllm-openai@sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (vLLM 0.25.1) | +| Engine | Private validation image at `sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (vLLM 0.25.1) | | LMCache | Client wheel 0.5.3 CUDA 12.9, test-only runtime-owner overlay; standalone sidecar `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b` | | Host-only TP=1 | Stored and retrieved 1,024 external tokens after clearing only vLLM's local prefix cache. | | Host-only TP=2 | Both ranks registered and retrieved correctly on one node. | @@ -704,9 +704,9 @@ remain until Phase 7. Physical removal is outside this phase. The Phase 0 owner audit and Phase 5 repository inventory found zero external consumers and zero installed legacy objects. This evidence covers this -repository and the recorded owner audit, not every organization source or OCI -cluster. Because no input population appeared, migration tooling and Phase 6 -were not activated. +repository and the recorded owner audit, not every organization source or +private validation cluster. Because no input population appeared, migration +tooling and Phase 6 were not activated. LMCacheServer and Mooncake are never translated to Redis automatically; the operator must choose host-only MP, Redis, or a future typed adapter. diff --git a/docs/design/lmcache-server-persistence.md b/docs/design/lmcache-server-persistence.md index da62c4f0..91f1658d 100644 --- a/docs/design/lmcache-server-persistence.md +++ b/docs/design/lmcache-server-persistence.md @@ -8,7 +8,7 @@ Status: locked · Scope: managed-backend durability (`CacheBackend`) > Redis. Do not treat the providers below as current defaults and do not map > them automatically to Redis; the operator must choose the desired L3 semantics. -## Decision +## Current decision `CacheBackend.spec.storage` — and the nested `storage.pvc.*` plus the `status.capacity` field — is **retired at `v1alpha1`**. Durability of a managed @@ -17,17 +17,14 @@ per-`CacheBackend` volume knob: - Omitting canonical `spec.remoteStorage` selects an engine-local host tier and provisions no provider workload. -- Historically, the managed **in-memory `lm://` LMCache server** - (`spec.remoteStorage.provider: LMCacheServer`) is the simple shared tier. It - keeps KV in process memory; it is not durable and does not persist across pod - restarts. -- Historically, the managed **Mooncake provider** - (`spec.remoteStorage.provider: Mooncake`) is the durable / shared / scalable - path: a network-addressable store the engine reaches over the - `mooncakestore://` remote wire. See - [Mooncake provider configuration](cachebackend-api.md#mooncake-provider-configuration). +- `spec.remoteStorage.provider: Redis` is the only current remote tier. Managed + ownership renders a single Redis Deployment and Service; external ownership + binds the declared endpoint without creating a workload. +- Removed `LMCacheServer` and Mooncake objects are not translated to Redis. + Reintroducing either technology requires a separately validated typed MP + adapter and an explicit operator migration choice. -## Why a local PVC cannot honestly back the `lm://` server +## Historical rationale: why a local PVC could not back the removed IP server An investigation into LMCache's storage model found **no mechanism by which a network-addressable, per-`CacheBackend` LMCache server persists KV to a local @@ -47,18 +44,21 @@ PVC**: implementation creates one directly scheduled server Pod per active engine node and CacheBackend; multiple pools on one node require disjoint host ports. -MP-mode is thus incompatible with this project's per-backend Deployment + -ClusterIP, engines-anywhere model. +That finding invalidated the old per-backend Deployment + ClusterIP model. The +current typed MP design instead uses a PodLocal native sidecar or one directly +scheduled NodeLocal server per active engine node; neither exposes the MP CUDA +data plane through a load-balanced Service. ## Consequences - `spec.storage{,.pvc}` + `status.capacity` were removed as a category error: the Kubernetes-side PVC plumbing could be provisioned, but could never honestly back the in-memory server. -- The historical durable/shared recommendation was the **Mooncake backend**. Its - managed workload lifecycle lives in the provider adapter - (`internal/adapters/builtin/storage/mooncake.go`), while the vLLM runtime adapter - (`internal/adapters/builtin/runtime/vllm_lmcache.go`) owns engine wiring. +- Current managed Redis lifecycle lives in + `internal/adapters/builtin/storage/redis.go`. Common typed MP rendering lives + in `internal/adapters/builtin/runtime/lmcache_mp_renderer.go` and + `lmcache_mp_nodelocal.go`; the SGLang and vLLM adapters own only their + engine-specific launch surfaces. - **Generalizable rule:** surface a `max*` / storage / quota field on a CRD only when the cache plane **authoritatively owns** the resource being limited. When it does not, omit the field or express the capability as a backend choice diff --git a/docs/design/sglang-lmcache-mp-mode.md b/docs/design/sglang-lmcache-mp-mode.md index b59c1e82..d4b8d0b5 100644 --- a/docs/design/sglang-lmcache-mp-mode.md +++ b/docs/design/sglang-lmcache-mp-mode.md @@ -116,109 +116,43 @@ tier is `resp` (Redis; simplest, proven), `s3`, `mooncake_store` (the RDMA path) or `p2p` (peer discovery). `resp` config schema (`RESPL2AdapterConfig`): `{"type":"resp","host":,"port":,"num_workers":8,"username":"","password":""}`. -## The three pieces, and how they map onto the interface - -A working `(sglang, LMCache)` deployment needs three things: - -1. **Engine wire** — `--enable-lmcache`, `LMCACHE_USE_EXPERIMENTAL=True`, - `--lmcache-config-file `; the file carries `mp_host`/`mp_port`/`chunk_size`. -2. **A node-local MP worker** reachable at `mp_host:mp_port`, co-located with the - engine (shared-memory data path), configured with the shared `--l2-adapter`. -3. **A shared L2 store** (Redis) the worker offloads to, reachable cluster-wide. - -The `KVCacheRuntimeAdapter` interface already accommodates all three **without a -new method**: - -### `ResolveCacheServer` → the shared L2 (Redis) - -The reconciler wraps the returned `(*PodSpec, *Service)` into one Deployment + -Service owned by the CR. For SGLang, that workload becomes the **shared Redis L2**, -with three constraints the design must honor: - -- **Pinned image** — a digest/tag tracked in `VERSIONS.md`, consistent with the - lmcache-server image-pin policy, never a floating `redis` tag. -- **Single replica (enforced) — specific to the SGLang Redis L2.** A plain Redis is - not clustered; multiple pods behind one Service would shard requests across - independent key spaces and silently partition the L2. So **this** backend is - **clamped to one replica** and HPA is not attached: admission rejects - `spec.replicas>1` / `spec.autoscaling` for `(sglang, LMCache)` - (`rejectSGLangRedisL2ScaleOut`), and the reconciler clamps as a backstop for - grandfathered objects (`clampSingletonReplicas`). This is **not** shared with - vLLM's `lm://` lmcache-server, which is an ordinary pod-network workload that - **does** scale out and autoscale — the singleton rule is scoped to the pair whose - managed server is a non-clustered Redis (and to the host-network Mooncake master, - for a different reason). A genuinely clustered/HA Redis is an operator-provided or - future option, out of scope for the managed default. -- **Bounded memory.** `--maxmemory-policy allkeys-lru` only evicts once - `--maxmemory` is set; without it Redis grows until the container is OOM-killed. - The render derives `--maxmemory` from the pod's memory limit (with headroom), - falling back to an explicit bounded default — so LRU eviction, not the OOM - killer, reclaims space. - -It listens on ClusterIP `:6379` and `status.endpoint` becomes the Redis Service -DNS. The provider-owned Redis renderer replaces the provider-owned `lm://` -lmcache-server renderer for the SGLang pair only — vLLM keeps `lm://`. Redis is -a shared, network-addressable store that fits the one-Service, engines-anywhere -model exactly (unlike Mooncake's mesh), so **no `hostNetwork` is required for -the L2**. - -### `InjectEngineConfig` → MP-worker native sidecar (writes its own config) + engine wire - -The mutating Pod webhook already adds volumes, init containers, and sidecar -containers to engine pods. For SGLang it adds, to the engine pod: - -- **the MP config file** — `/etc/lmcache/config.yaml` (`chunk_size`, - `mp_host: 127.0.0.1`, `mp_port`) in a shared `emptyDir` (`lmcache-config`). - **As built, the worker sidecar writes this itself** and then `exec`s the MP - server, rather than a separate `lmcache-config` init container doing it: the two - always agree on `mp_port` because one process renders both sides, and the worker's - `startupProbe` already gates the engine on the server listening — which implies the - file exists, since the `exec` happens after the write. A separate init container - would have been a second place to keep in sync for no added ordering guarantee. - No ConfigMap needed (the webhook cannot create cluster resources; the values are - static and small). -- **native sidecar `lmcache-mp-worker`** — runs the upstream-documented worker CLI - `python3 -m lmcache.v1.multiprocess.server --host 127.0.0.1 --port - --chunk-size --l1-size-gb --eviction-policy LRU - --l2-adapter '{"type":"resp","host":,"port":}'` - (the documented `lmcache server` subcommand is the equivalent entrypoint; the - rendered wire uses the `python3 -m` form, which is what validation exercised). - `` is the Redis address passed to `InjectEngineConfig`. Its - **image defaults to the engine's own image (and should be digest-pinned in - production)**, keeping it version-aligned with the engine's LMCache connector — - the two speak the LMCache MP wire (ZMQ + shared memory), so a version skew between - worker and engine is a correctness hazard, and defaulting to the same image makes - the aligned case the zero-config one. `spec.lmCache.workerImage` overrides it, at - which point the alignment (and the digest pin) is the operator's to maintain; the - tuple is tracked in - `VERSIONS.md` alongside the engine image (validation used the engine's own - `pip install lmcache`→0.5.1, so the simplest pin is the same image and `lmcache` - version for both). Mounts the shared `/dev/shm` - (`emptyDir{medium: Memory, sizeLimit ≥ l1-size}`) and `/etc/lmcache`. **Startup - ordering matters** — the engine dials the worker at launch, and K8s does not - order ordinary containers within a pod. So this is a **native sidecar** (a - `restartPolicy: Always` entry in `initContainers` — beta and on-by-default since - K8s 1.29, stable since 1.33) with a `startupProbe`. The ZMQ port binds - `127.0.0.1`, which a pod-IP-targeted `tcpSocket`/`httpGet` probe cannot reach, so - the probe is either an **`exec`** loopback check (runs inside the container) or — - cleaner — an **`httpGet` on the worker's HTTP management endpoint (`:8080`)**, - which `lmcache server` exposes and which can bind the pod interface; Phase 2 - picks whichever the pinned build supports. Native sidecars start and gate ready - **before** the main engine container, so the worker is listening when the engine - connects. (An ordinary sidecar would race the engine.) The adapter's minimum is - K8s ≥ 1.29. Fail-open interaction is resolved below. -- **engine container** — add `--enable-lmcache`, `--lmcache-config-file - /etc/lmcache/config.yaml`, `LMCACHE_USE_EXPERIMENTAL=True`, - `INFERENCECACHE_FAIL_OPEN`; mount the shared `/dev/shm` + `/etc/lmcache`. **Drop** - the MP-ignored `LMCACHE_REMOTE_URL` / `LMCACHE_REMOTE_SERDE` / `LMCACHE_LOCAL_CPU` / - `LMCACHE_MAX_LOCAL_CPU_SIZE` env. - -`mp_host=127.0.0.1` works because the worker is a **same-pod sidecar** — it shares -the engine's network namespace, so ZMQ over loopback reaches it and the shared -`/dev/shm` `emptyDir` gives the data path. This is the key divergence from -Mooncake: Mooncake needs `hostNetwork` on the engine (its mesh dials real host -IPs on dynamic ports); SGLang's MP worker is in-pod, so the engine stays on the -pod network. +## Current implementation mapping + +A current `(SGLang, LMCache)` deployment has three independently owned pieces: + +1. The runtime adapter mutates the engine Pod through `InjectEngineConfig`. +2. The common MP renderer creates either a PodLocal native sidecar or an + on-demand NodeLocal server Pod from the typed `spec.lmCache` declaration. +3. The storage adapter optionally renders or binds Redis from + `spec.remoteStorage`; omitting it is the supported host-only configuration. + +The removed `ResolveCacheServer` path is not part of the current runtime +interface. Managed Redis lifecycle belongs to +`internal/adapters/builtin/storage/redis.go`; runtime-specific engine wiring +belongs to `sglang_lmcache.go`; shared MP server rendering belongs to +`lmcache_mp_renderer.go` and `lmcache_mp_nodelocal.go`. + +For **PodLocal**, the webhook injects a native sidecar named +`lmcache-mp-server`. Its image, port, L1 capacity, worker count, and resources +come from `spec.lmCache.podLocal.server`; the image never defaults from the +engine container. The server uses the `lmcache server` entrypoint, mounts the +shared memory volume, and writes the engine-specific config file. SGLang gets +`--enable-lmcache`, `--lmcache-config-file`, and +`LMCACHE_USE_EXPERIMENTAL=True`; the native sidecar must be Ready before the +engine starts. + +For **NodeLocal**, the controller creates one direct server Pod per active +engine node using exact-node affinity, host networking, declared host ports, +and a UID-scoped `--shm-name`. The engine receives a same-node endpoint derived +from the Downward API and a startup gate that verifies server ownership, +generation, shared-memory identity, and health. No load-balanced MP Service is +created. + +Redis is an optional remote tier for either topology. Managed Redis is a +single-replica Deployment and Service; external Redis publishes the declared +endpoint without creating a workload. Its state is reported under +`status.remoteStorage`, independently from required connector health under +`status.connector` and `ConnectorReady`. ### Fail-open semantics (resolving the startup-gate tension) diff --git a/docs/reference-stack/GPU-RUNBOOK.md b/docs/reference-stack/GPU-RUNBOOK.md index e98b5733..e470a857 100644 --- a/docs/reference-stack/GPU-RUNBOOK.md +++ b/docs/reference-stack/GPU-RUNBOOK.md @@ -5,8 +5,8 @@ available, and how to size it: **GPU memory, card count, tensor-parallelism, and host resources**. The stack is cloud-neutral — it needs an NVIDIA GPU advertising `nvidia.com/gpu`, -nothing more. §4 gives a concrete OCI-shape mapping as one worked example; any -equivalent NVIDIA card on any cloud or on-prem works the same. +nothing more. Select any cloud or on-prem node whose card count, VRAM, host +memory, and interconnect satisfy the requirements below. --- @@ -76,31 +76,11 @@ Rules of thumb: | `/dev/shm` | ≥ typed L1 + 1Gi | Shared by the engine and injected MP server; the reference uses 8Gi. | | Local disk | model size × 1.5 | HF weight cache. The host-only reference does not claim a local-disk LMCache tier. | | Network | 100 Gb+ RDMA for multi-node | Only if you later shard across nodes; single-node TP uses NVLink. | -| Driver/runtime | NVIDIA driver + Container Toolkit; `nvidia` default Docker runtime | So kind/OKE pods can request `nvidia.com/gpu`. | +| Driver/runtime | NVIDIA driver + Container Toolkit; `nvidia` default Docker runtime | So local or managed-cluster pods can request `nvidia.com/gpu`. | --- -## 4. Worked example — OCI GPU shapes - -One concrete cloud mapping (Oracle Cloud Infrastructure). Any equivalent NVIDIA -card on another cloud or on-prem works the same. GPU memory **per card**: A10 = -24 GB, L40S = 48 GB, A100 = 80 GB (also a 40 GB variant), H100 = 80 GB, H200 = -141 GB. - -| Target | OCI shape | Cards × VRAM | Good for | -|---|---|---|---| -| **This reference (recommended)** | `VM.GPU.A10.1` | 1 × 24 GB | 8B, single card, cheapest | -| Single card with headroom | `BM.GPU.L40S.4` (use 1 GPU) | 4 × 48 GB | 8B–34B comfortably; room for LMCache | -| Single bigger model | `VM.GPU.A100.1` / `VM.GPU.H100.1` | 1 × 80 GB | up to ~34B, or 70B quantized | -| 70B BF16 (TP) | `BM.GPU4.8` / `BM.GPU.A100-v2.8` | 8 × 40/80 GB (use 4, NVLink) | `--tensor-parallel-size 4` | -| Largest / fastest | `BM.GPU.H100.8` / `BM.GPU.H200.8` | 8 × 80/141 GB | 70B–100B+, full-node TP | - -For the **8B reference**, a single 24 GB card (e.g. `VM.GPU.A10.1`) is the -cheapest option. Pick a bare-metal multi-GPU shape only when you need TP ≥ 2. - ---- - -## 5. Deploy (once the GPU node is up) +## 4. Deploy (once the GPU node is up) Builds on [`README.md`](README.md) "Deploy and test on a GPU". Summary: @@ -139,7 +119,7 @@ Then run the verification in [`README.md`](README.md) ("What success looks like"): subscribe with `scripts/kv_events_subscriber.py` and fire `scripts/prefix_cache_hit_test.sh`. -## 6. Sizing-related failure cheatsheet +## 5. Sizing-related failure cheatsheet | Symptom | Likely cause | Fix | |---|---|---| diff --git a/docs/reference-stack/VERSIONS.md b/docs/reference-stack/VERSIONS.md index 30bf370e..5e7a827b 100644 --- a/docs/reference-stack/VERSIONS.md +++ b/docs/reference-stack/VERSIONS.md @@ -29,7 +29,7 @@ Phase 3 and Phase 4 ran on 2026-08-10/11 in the SJC development environment: | Path | Engine evidence | LMCache evidence | Result | |---|---|---|---| | SGLang TP=1 | `docker.io/lmsysorg/sglang@sha256:920df39109c60429b0a23eaacfd2786fcf1595c12f3ca4fc6e153b2abe34865f` (`0.5.13.post1-cu129`) | Client wheel 0.5.3 CUDA 12.9 via a test-only runtime-owner overlay; standalone `sha256:0df30fc70a7d689e1f12823789208a0ee8ef31537316eba6a4c2fa83b0abe61b` | Host-only store/retrieve, bounded L1 eviction, events/status, and managed-Redis replacement-Pod retrieval passed. | -| vLLM TP=1/2 | `us-sanjose-1.ocir.io/idqj093njucb/vllm-openai@sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (`0.25.1`) | Client wheel 0.5.3 CUDA 12.9 via a test-only runtime-owner overlay; same validation standalone digest | Host-only TP=1/2 retrieval, events/status, shared-memory budget, native extension checks, and supplemental Redis replacement-Pod retrieval passed. | +| vLLM TP=1/2 | Private validation image at `sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a` (`0.25.1`) | Client wheel 0.5.3 CUDA 12.9 via a test-only runtime-owner overlay; same validation standalone digest | Host-only TP=1/2 retrieval, events/status, shared-memory budget, native extension checks, and supplemental Redis replacement-Pod retrieval passed. | The runtime-owner wheel overlays were validation scaffolding, not authority for `CacheBackend` to mutate an engine image. These records prove the controller diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index f2f8ae62..776e72a3 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -57,6 +57,9 @@ collect_diagnostics() { kubectl -n "$SYSTEM_NAMESPACE" get all -o wide >"$LOG_DIR/system.txt" 2>&1 || true kubectl -n "$SYSTEM_NAMESPACE" get events --sort-by=.lastTimestamp >"$LOG_DIR/events.txt" 2>&1 || true kubectl -n "$SYSTEM_NAMESPACE" logs deployment/inference-cache-controller-manager --all-containers >"$LOG_DIR/controller.log" 2>&1 || true + kubectl -n "$SMOKE_NAMESPACE" get all -o wide >"$LOG_DIR/smoke-system.txt" 2>&1 || true + kubectl -n "$SMOKE_NAMESPACE" get pods -o json >"$LOG_DIR/smoke-pods.json" 2>&1 || true + kubectl -n "$SMOKE_NAMESPACE" get events --sort-by=.lastTimestamp >"$LOG_DIR/smoke-events.txt" 2>&1 || true } cleanup() { @@ -383,6 +386,7 @@ for _ in $(seq 1 60); do sleep 1 done [ -n "${engine_node:-}" ] && [ -n "${server_name:-}" ] || fail "scheduled engine did not create an on-demand NodeLocal server" +kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o json >"$LOG_DIR/node-local-server.json" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-target-node}')" = "$engine_node" ] \ || fail "NodeLocal server target does not match the engine-selected node" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.hostNetwork}')" = "true" ] \ @@ -397,11 +401,11 @@ node_local_shm_name="lmcache_l1_pool_inferencecache_${node_local_backend_uid}" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-shm-name}')" = "$node_local_shm_name" ] \ || fail "NodeLocal server does not carry its UID-scoped shared-memory identity" node_local_shm_arg="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" \ - -o jsonpath='{range .spec.containers[0].args[*]}{.}{"\n"}{end}' | \ + -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].args[*]}{@}{"\n"}{end}' | \ awk 'previous == "--shm-name" { print; exit } { previous = $0 }')" [ "$node_local_shm_arg" = "$node_local_shm_name" ] \ || fail "NodeLocal server does not pass its UID-scoped --shm-name: $node_local_shm_arg" -node_local_ports="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{range .spec.containers[0].ports[*]}{.containerPort}:{.hostPort}{" "}{end}')" +node_local_ports="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].ports[*]}{.containerPort}:{.hostPort}{" "}{end}')" [ "$node_local_ports" = "5556:5556 8081:8081 " ] || fail "NodeLocal host ports were not declared: $node_local_ports" node_affinity_target="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchFields[0].values[0]}')" [ "$node_affinity_target" = "$engine_node" ] || fail "server does not use scheduler-bound exact-node affinity" diff --git a/hack/verify-samples/admission_test.go b/hack/verify-samples/admission_test.go index 2a033896..a5c3aa1e 100644 --- a/hack/verify-samples/admission_test.go +++ b/hack/verify-samples/admission_test.go @@ -12,6 +12,8 @@ import ( "testing" "time" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -140,6 +142,23 @@ func TestVerifySamplesAdmissionEndToEnd(t *testing.T) { Spec: cachev1alpha1.CacheBackendSpec{ Runtime: cachev1alpha1.CacheBackendRuntimeVLLM, Type: cachev1alpha1.CacheBackendTypeLMCache, + LMCache: &cachev1alpha1.LMCacheEngineSpec{ + Topology: cachev1alpha1.LMCacheTopologyPodLocal, + PodLocal: &cachev1alpha1.LMCachePodLocalSpec{ + Server: &cachev1alpha1.LMCachePodLocalServerSpec{ + Image: "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + Port: 6500, + L1Capacity: resource.MustParse("1Gi"), + MaxWorkers: 1, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("2Gi")}, + }, + }, + }, + }, }, } if err := cl.Create(ctx, good, client.DryRunAll); err != nil { diff --git a/internal/adapters/builtin/runtime/runtime_helpers.go b/internal/adapters/builtin/runtime/runtime_helpers.go index 52c6ee8e..cef55ae5 100644 --- a/internal/adapters/builtin/runtime/runtime_helpers.go +++ b/internal/adapters/builtin/runtime/runtime_helpers.go @@ -85,12 +85,11 @@ func UpsertArgPair(args []string, flag, value string) []string { } return append(args, value) case strings.HasPrefix(arg, prefix): + args = append(args, "") + copy(args[i+2:], args[i+1:]) args[i] = flag - out := make([]string, 0, len(args)+1) - out = append(out, args[:i+1]...) - out = append(out, value) - out = append(out, args[i+1:]...) - return out + args[i+1] = value + return args } } return append(args, flag, value) diff --git a/internal/controller/cachebackend_kvevent_gate_test.go b/internal/controller/cachebackend_kvevent_gate_test.go index 1f0a87ec..b2a55a6b 100644 --- a/internal/controller/cachebackend_kvevent_gate_test.go +++ b/internal/controller/cachebackend_kvevent_gate_test.go @@ -6,6 +6,7 @@ package controller import ( "context" + "strconv" "testing" "time" @@ -20,6 +21,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" + "github.com/cachebox-project/inference-cache/internal/enginebinding" ) // gatedLMCacheBackend is a managed LMCache backend WITH the KV-event readiness @@ -28,6 +30,9 @@ import ( func gatedLMCacheBackend(name, ns string) *cachev1alpha1.CacheBackend { cb := lmcacheBackend(name, ns) delete(cb.Annotations, annotationRequireKVEvents) + cb.Spec.EngineSelector = &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + cachev1alpha1.CacheBackendDomainLabel: name, + }} return cb } @@ -57,6 +62,7 @@ func setFirstAvailableAt(t *testing.T, cl client.Client, name, ns string, at tim // condition. func setDeploymentHTTPReady(t *testing.T, cl client.Client, name, ns string, availableSince time.Time) { t.Helper() + setPodLocalConnectorReady(t, cl, name, ns) var dep appsv1.Deployment if err := cl.Get(context.Background(), types.NamespacedName{Name: name, Namespace: ns}, &dep); err != nil { t.Fatalf("get deployment %s/%s: %v", ns, name, err) @@ -81,6 +87,79 @@ func setDeploymentHTTPReady(t *testing.T, cl client.Client, name, ns string, ava } } +// setPodLocalConnectorReady creates the engine-side state that typed MP +// readiness requires. Envtest has no inference owner, mutating webhook, or +// kubelet, so the fixture must explicitly model their persisted output: a +// selected Pod with ownership stamps, a running native sidecar, and PodReady. +func setPodLocalConnectorReady(t *testing.T, cl client.Client, name, ns string) { + t.Helper() + ctx := context.Background() + var backend cachev1alpha1.CacheBackend + if err := cl.Get(ctx, types.NamespacedName{Name: name, Namespace: ns}, &backend); err != nil { + t.Fatalf("get CacheBackend %s/%s for connector fixture: %v", ns, name, err) + } + if !isTypedLMCachePodLocal(&backend) || backend.Spec.EngineSelector == nil || len(backend.Spec.EngineSelector.MatchLabels) == 0 { + return + } + + key := types.NamespacedName{Name: name + "-ready-engine", Namespace: ns} + pod := &corev1.Pod{} + err := cl.Get(ctx, key, pod) + if client.IgnoreNotFound(err) != nil { + t.Fatalf("get connector fixture Pod %s/%s: %v", ns, key.Name, err) + } + if err != nil { + pod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, Namespace: ns, + Labels: backend.Spec.EngineSelector.MatchLabels, + Annotations: map[string]string{ + enginebinding.AnnotationInjectedBy: ns + "/" + name, + enginebinding.AnnotationInjectedByUID: string(backend.UID), + enginebinding.AnnotationInjectedGeneration: strconv.FormatInt(backend.Generation, 10), + }, + }, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "vllm", Image: "registry.example.com/vllm:test"}}}, + } + if err := cl.Create(ctx, pod); err != nil { + t.Fatalf("create connector fixture Pod %s/%s: %v", ns, key.Name, err) + } + } else { + before := pod.DeepCopy() + pod.Labels = backend.Spec.EngineSelector.MatchLabels + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[enginebinding.AnnotationInjectedBy] = ns + "/" + name + pod.Annotations[enginebinding.AnnotationInjectedByUID] = string(backend.UID) + pod.Annotations[enginebinding.AnnotationInjectedGeneration] = strconv.FormatInt(backend.Generation, 10) + if err := cl.Patch(ctx, pod, client.MergeFrom(before)); err != nil { + t.Fatalf("patch connector fixture Pod %s/%s metadata: %v", ns, key.Name, err) + } + } + + before := pod.DeepCopy() + pod.Status.Phase = corev1.PodRunning + pod.Status.InitContainerStatuses = []corev1.ContainerStatus{{ + Name: lmCacheMPServerStatusContainerName, Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := cl.Status().Patch(ctx, pod, client.MergeFrom(before)); err != nil { + t.Fatalf("patch connector fixture Pod %s/%s status: %v", ns, key.Name, err) + } + + // Seed the production connector projection before the next full reconcile. + // The reconciler deliberately refreshes this independent status writer after + // readiness aggregation, so a single-shot envtest otherwise observes the + // previous connector verdict for one cycle. + if err := cl.Get(ctx, types.NamespacedName{Name: name, Namespace: ns}, &backend); err != nil { + t.Fatalf("refresh CacheBackend %s/%s for connector projection: %v", ns, name, err) + } + fixtureReconciler := &CacheBackendReconciler{Client: cl, APIReader: cl} + fixtureReconciler.refreshLMCacheMPConnectorStatus(ctx, &backend) +} + // patchLastEventAt simulates the CacheIndex poller writing a fresh KV-event // timestamp into status.indexParticipation via the status subresource — the // exact path the poller uses, and the signal the gate reads. @@ -268,6 +347,10 @@ func TestIntegrationKVEventReadinessGate(t *testing.T) { if err := k8s.Update(ctx, live); err != nil { t.Fatalf("update firstEventTimeout: %v", err) } + // Model the engine rollout/reinjection that authenticates the Pod for + // the new CacheBackend generation; this test is about the sticky + // KV-event verdict, not connector rollout behavior. + setPodLocalConnectorReady(t, k8s, "cache", ns) reconcile(t, r, "cache", ns) got := getBackend(t, r, "cache", ns) @@ -315,7 +398,7 @@ func TestIntegrationKVEventReadinessGate(t *testing.T) { } }) - t.Run("ExternalBypassesGateEntirely", func(t *testing.T) { + t.Run("ExternalRemoteStorageBypassesKVEventGate", func(t *testing.T) { ns := freshNS(t, k8s) cb := &cachev1alpha1.CacheBackend{ ObjectMeta: metav1.ObjectMeta{Name: "ext", Namespace: ns}, @@ -325,23 +408,26 @@ func TestIntegrationKVEventReadinessGate(t *testing.T) { LMCache: lmcacheBackend("fixture", ns).Spec.LMCache.DeepCopy(), RemoteStorage: externalRedisStorage("external.example.svc:6379"), Integration: &cachev1alpha1.CacheBackendIntegrationSpec{Role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite}, + EngineSelector: &cachev1alpha1.CacheBackendEngineSelector{MatchLabels: map[string]string{ + cachev1alpha1.CacheBackendDomainLabel: "ext", + }}, }, } if err := k8s.Create(ctx, cb); err != nil { t.Fatalf("create: %v", err) } + setPodLocalConnectorReady(t, k8s, "ext", ns) reconcile(t, r, "ext", ns) got := getBackend(t, r, "ext", ns) if got.Status.RemoteStorage == nil || got.Status.RemoteStorage.Endpoint != "external.example.svc:6379" { t.Fatalf("remoteStorage status = %+v, want mirrored external endpoint", got.Status.RemoteStorage) } - // External never enters the KV-event gate: readiness comes from - // admission accepting the endpoint (reason ExternalEndpointAccepted), - // not from a KV event. The gate's reasons and latch must never appear. + // External Redis never enters the KV-event gate. The required local MP + // connector still gates readiness independently of the remote tier. ready := findCondition(got.Status.Conditions, conditionTypeReady) - if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != "ExternalEndpointAccepted" { - t.Fatalf("Ready = %+v, want True/ExternalEndpointAccepted (endpoint-driven, not gated)", ready) + if ready == nil || ready.Status != metav1.ConditionTrue || ready.Reason != reasonConnectorReady { + t.Fatalf("Ready = %+v, want True/%s (connector-ready, not KV-event-gated)", ready, reasonConnectorReady) } if deg := findCondition(got.Status.Conditions, conditionTypeDegraded); deg != nil { t.Fatalf("Degraded = %+v, want absent for External", deg) diff --git a/internal/controller/cachebackend_schema_trim_integration_test.go b/internal/controller/cachebackend_schema_trim_integration_test.go index e886efd7..ba0fe1e6 100644 --- a/internal/controller/cachebackend_schema_trim_integration_test.go +++ b/internal/controller/cachebackend_schema_trim_integration_test.go @@ -79,6 +79,19 @@ func TestIntegrationCacheBackendSchemaTrim(t *testing.T) { if err := unstructured.SetNestedStringMap(u.Object, map[string]string{"app.kubernetes.io/name": "vllm"}, "spec", "engineSelector", "matchLabels"); err != nil { t.Fatalf("set spec.engineSelector: %v", err) } + if err := unstructured.SetNestedMap(u.Object, map[string]any{ + "topology": "PodLocal", + "podLocal": map[string]any{"server": map[string]any{ + "image": "registry.example.com/lmcache@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "port": int64(6500), "l1Capacity": "1Gi", "maxWorkers": int64(1), + "resources": map[string]any{ + "requests": map[string]any{"cpu": "1", "memory": "2Gi"}, + "limits": map[string]any{"memory": "2Gi"}, + }, + }}, + }, "spec", "lmCache"); err != nil { + t.Fatalf("set spec.lmCache: %v", err) + } return u } From 5411c5c35df115fe5b166b392213f84cc8b057bf Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 12 Aug 2026 22:44:31 -0700 Subject: [PATCH 10/13] Isolate NodeLocal shared memory by backend UID Signed-off-by: Yue Sun --- config/samples/README.md | 2 +- ...chebackend-sglang-nodelocal-host-only.yaml | 3 +- ...cachebackend-vllm-nodelocal-host-only.yaml | 5 +- docs/concepts/cachebackend-engine-binding.md | 14 ++-- docs/design/cachebackend-api.md | 29 ++++--- .../lmcache-multiprocess-migration-roadmap.md | 84 ++++++++++++------- docs/design/lmcache-server-persistence.md | 6 +- docs/quickstart.md | 6 +- .../scripts/default_install_smoke.sh | 11 ++- .../builtin/runtime/lmcache_mp_nodelocal.go | 42 +++++++--- .../runtime/lmcache_mp_nodelocal_test.go | 50 ++++++++++- .../cachebackend_lmcache_mp_status.go | 7 +- .../cachebackend_lmcache_mp_status_test.go | 20 ++++- .../cachebackend_lmcache_nodelocal.go | 38 +++++++-- .../cachebackend_mp_lifecycle_test.go | 46 +++++++++- ...cachebackend_nodelocal_integration_test.go | 6 +- 16 files changed, 289 insertions(+), 80 deletions(-) diff --git a/config/samples/README.md b/config/samples/README.md index 7bf107d5..8f178378 100644 --- a/config/samples/README.md +++ b/config/samples/README.md @@ -27,7 +27,7 @@ multi-tenant, Namespaces): available for [vLLM](cachebackend-vllm-nodelocal-host-only.yaml) and [SGLang](cachebackend-sglang-nodelocal-host-only.yaml); the inference system owns their placement and they opt into server host networking plus shared - host `/dev/shm`. + the backend UID's host `/dev/shm/inference-cache/` directory. `EventsOnly` intentionally carries no LMCache data plane. ## Recipe catalog diff --git a/config/samples/cachebackend-sglang-nodelocal-host-only.yaml b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml index 4299fd2b..9ff21f9f 100644 --- a/config/samples/cachebackend-sglang-nodelocal-host-only.yaml +++ b/config/samples/cachebackend-sglang-nodelocal-host-only.yaml @@ -8,7 +8,8 @@ # nodes and never rewrites engine placement. The 5556/8081 host-port pair is # intentionally disjoint from the sibling vLLM sample if both backends have # engines on the same node. Set scheduling.runtimeClassName when the engine's -# runtime does not provide the server's required NVIDIA visibility. +# runtime does not provide the server's required NVIDIA visibility. The server +# and engines mount only this backend UID's host SHM directory as `/dev/shm`. apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: diff --git a/config/samples/cachebackend-vllm-nodelocal-host-only.yaml b/config/samples/cachebackend-vllm-nodelocal-host-only.yaml index f536a8a6..f6ea5cc8 100644 --- a/config/samples/cachebackend-vllm-nodelocal-host-only.yaml +++ b/config/samples/cachebackend-vllm-nodelocal-host-only.yaml @@ -5,8 +5,9 @@ # Typed vLLM NodeLocal LMCache MP with one on-demand shared server on every node # that actually hosts a selected engine. The inference system remains the # scheduling authority; CacheBackend does not select nodes or change engine -# placement. The server uses hostNetwork and host /dev/shm; node firewall policy -# must restrict the unauthenticated MP/HTTP host ports. CacheBackend does not +# placement. The server uses hostNetwork; the server and engines mount only this +# backend UID's host /dev/shm/inference-cache/ directory. Node firewall +# policy must restrict the unauthenticated MP/HTTP host ports. CacheBackend does not # install LMCache into or replace the engine image. If NVIDIA is not the # engine/server runtime, set nodeLocal.scheduling.runtimeClassName. apiVersion: inferencecache.io/v1alpha1 diff --git a/docs/concepts/cachebackend-engine-binding.md b/docs/concepts/cachebackend-engine-binding.md index 87237cb1..fc06259e 100644 --- a/docs/concepts/cachebackend-engine-binding.md +++ b/docs/concepts/cachebackend-engine-binding.md @@ -13,7 +13,8 @@ NodeLocal topology. At Pod CREATE, the webhook: 1. finds the one matching CacheBackend in the Pod's namespace; 2. selects the runtime-specific MP adapter; 3. injects the vLLM or SGLang connector launch surface plus either a PodLocal - native sidecar or a NodeLocal same-node startup gate and host `/dev/shm`; + native sidecar or a NodeLocal same-node startup gate and the backend UID's + host SHM directory mounted as `/dev/shm`; 4. optionally binds the MP server to a Redis L3; and 5. stamps `inferencecache.io/injected-by` and `inferencecache.io/injected-by-uid`. @@ -32,10 +33,13 @@ engine startup until `/config` and `/healthcheck` verify the same name/UID/generation and live server configuration. Each CacheBackend UID also derives an explicit `lmcache_l1_pool_inferencecache_` POSIX SHM name; the gate verifies both the declared and effective live name before starting the -engine. This prevents accidental unlink/rebind between co-located pools but is -ownership verification, not cryptographic authentication, so the -host-network/server pool and node-wide `/dev/shm` still require one trusted -tenant domain. +engine. The server and engines mount only +`/dev/shm/inference-cache/` from the host as their container +`/dev/shm`, so normally behaving co-located pools do not see one another's SHM +objects. This remains ownership isolation rather than cryptographic +authentication: host root, privileged Pods, or processes mounting the parent +directory can bypass it, so the host-network/server pool still requires one +trusted tenant domain. When the final selected engine leaves a node, the server enters the configured `idleRetentionSeconds` window instead of being coupled to that engine Pod's restart. Demand returning during the window clears the idle marker and reuses diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 88da9718..9f0c0d49 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -220,9 +220,10 @@ serving if its image does not contain a compatible LMCache client/API; admission does not pull, execute, or otherwise introspect the engine image. For PodLocal the webhook injects a digest-pinned `lmcache-mp-server` native -sidecar. For NodeLocal it preserves engine placement, mounts host `/dev/shm`, -and adds a blocking identity/health gate while the controller follows the -scheduled engine with a same-node server Pod. Both add the following vLLM +sidecar. For NodeLocal it preserves engine placement, mounts only the +backend-UID host SHM directory as `/dev/shm`, and adds a blocking +identity/health gate while the controller follows the scheduled engine with a +same-node server Pod. Both add the following vLLM launch contract: - `--kv-transfer-config` selects `LMCacheMPConnector` through @@ -258,12 +259,14 @@ for LMCache metadata and shared-memory allocator overhead. For NodeLocal, `l1Capacity` is instead one shared per-node budget. CacheBackend creation alone creates no server and never changes engine placement. After an injected engine has been scheduled, the controller owns one host-networked -server Pod for each distinct active engine node, declares the MP and FastAPI -listeners as host ports, and mounts the node's `/dev/shm` into both the server -and selected engine Pods. Exact node-name affinity sends the server through the -normal scheduler on the engine's node; `status.hostIP` prevents ClusterIP or -cross-node CUDA IPC. `maxGPUWorkers` must cover all selected engine instances on -one node. Every server receives the controller-derived +server Pod for each distinct active engine node and declares the MP and FastAPI +listeners as host ports. Both the server and selected engines mount only the +backend's `/dev/shm/inference-cache/` host directory as their +container `/dev/shm`; they do not mount the whole node SHM namespace. Exact +node-name affinity sends the server through the normal scheduler on the +engine's node; `status.hostIP` prevents ClusterIP or cross-node CUDA IPC. +`maxGPUWorkers` must cover all selected engine instances on one node. Every +server receives the controller-derived `lmcache_l1_pool_inferencecache_` through `--shm-name`; the engine gate verifies both the declared MP value and the effective L1 memory-manager value before startup. Different CacheBackend UIDs therefore do @@ -275,9 +278,11 @@ image-pull secrets, priority class, and scheduler unless optional are unauthenticated and host networking bypasses NetworkPolicy, so this topology requires one trusted tenant domain per pool plus node firewall controls. CacheBackend name/UID/generation verification detects wrong ownership but is -not cryptographic authentication, and unique names do not isolate hostile -processes that already have compatible access to host `/dev/shm`. Co-located -pools therefore remain limited to one trusted node domain. After the last +not cryptographic authentication. The UID-scoped mount prevents normal pool +processes from seeing another pool through their container `/dev/shm`, but does +not isolate host root, privileged Pods, or processes that independently mount +the parent host directory. Co-located pools therefore remain limited to one +trusted node domain. After the last selected engine leaves a node, `nodeLocal.idleRetentionSeconds` keeps the server and shared L1 warm for the configured window (300 seconds by default); new demand on that node reuses diff --git a/docs/design/lmcache-multiprocess-migration-roadmap.md b/docs/design/lmcache-multiprocess-migration-roadmap.md index 9d55c07c..767ac2fa 100644 --- a/docs/design/lmcache-multiprocess-migration-roadmap.md +++ b/docs/design/lmcache-multiprocess-migration-roadmap.md @@ -104,7 +104,7 @@ code lands. | D10 | Component lifecycle ownership is capability-specific. | The Pod-local MP process is kubelet-owned while remote L3 is independently managed; connector re-registration after an MP-process restart is a post-migration enhancement, not an MVP contract. | | D11 | Each supported vLLM integration explicitly identifies its MP connector implementation; the initial reference baseline uses the LMCache-shipped connector. | With vLLM 0.20 or newer, `LMCacheMPConnector` without a module path selects vLLM's built-in implementation. The initial adapter uses `kv_connector_module_path: lmcache.integration.vllm.lmcache_mp_connector` so the tested client tracks the pinned LMCache server protocol; a future adapter revision may validate a different implementation explicitly. | | D12 | CacheBackend never owns or rewrites the inference engine image. Engine images in validation matrices are reproducible fixtures only; CacheBackend digest-pins only cache components it injects or manages. | The inference system owns its runtime lifecycle. The selected adapter renders its engine-specific connector contract, while normal engine initialization is the authoritative compatibility check; tested images are neither an admission allowlist nor a mutation default. | -| D13 | Selecting `NodeLocal` explicitly opts the backend into one host-networked MP server per active engine node and host `/dev/shm` mounts in both server and selected engine Pods; engine Pods themselves remain off host networking and host IPC. | LMCache 0.5.3 requires node-visible networking and shared host memory for cross-Pod CUDA IPC. Keeping engine placement and networking under the inference system reduces coupling, while the topology choice and documented trust domain make the remaining host access explicit. | +| D13 | Selecting `NodeLocal` explicitly opts the backend into one host-networked MP server per active engine node. Server and selected engine Pods mount only `/dev/shm/inference-cache/` from the host as container `/dev/shm`; engine Pods themselves remain off host networking and host IPC. | LMCache 0.5.3 requires node-visible networking and shared host memory for cross-Pod CUDA IPC. A UID-scoped bind mount retains that path without exposing the entire node SHM namespace to normally behaving pool processes. Keeping engine placement and networking under the inference system reduces coupling, while the topology choice and documented trust domain make the remaining host access explicit. | ## Migration baseline (before Phase 1) @@ -915,9 +915,10 @@ The final contract is: server and L1 for reuse; expiry removes it, while zero requests immediate deletion. No Deployment, ReplicaSet, or DaemonSet owns these Pods. - **Host boundary:** Server Pods use `hostNetwork`, `ClusterFirstWithHostNet`, - host `/dev/shm`, the selected NVIDIA runtime without reserving allocatable - GPUs, and a restrictive container security context. Engine Pods remain off - host networking and host IPC. + the selected NVIDIA runtime without reserving allocatable GPUs, and a + restrictive container security context. Servers and selected engines mount + only the backend's `/dev/shm/inference-cache/` host directory as their + container `/dev/shm`; engines remain off host networking and host IPC. - **Endpoint and gate:** Engines derive the same-node address from Downward API `status.hostIP`. A blocking init gate requires healthy `/healthcheck` plus an exact `/config` match for namespace/name/UID/generation, ports, and chunk size @@ -930,15 +931,18 @@ The final contract is: namespace-unique `inferencecache.io/cache-domain` value own one server pool. CREATE and UPDATE reject non-canonical or duplicate ownership; Pod admission denies concurrent ambiguity. Every server receives the full UID-derived - `lmcache_l1_pool_inferencecache_` name through `--shm-name`; the name is - stable across same-UID generation/server replacement and distinct after - CacheBackend delete/recreate. Disjoint port pairs prevent network bind - conflicts, while UID-scoped names prevent accidental POSIX SHM unlink/rebind - between co-located pools. Idle-retained servers continue reserving their - ports and SHM budget until expiry. UID matching and a unique SHM name are - routing/ownership identities, not authentication: co-located pools must - remain inside one mutually trusted node domain, host firewall controls are - required, and NetworkPolicy does not isolate host-network listeners. + `lmcache_l1_pool_inferencecache_` name through `--shm-name`; the name and + UID host directory are stable across same-UID generation/server replacement + and distinct after CacheBackend delete/recreate. Disjoint port pairs prevent + network bind conflicts, while the UID-scoped name and mount prevent normal + co-located pools from accidentally unlinking/rebinding or seeing each + other's POSIX SHM objects. Idle-retained servers continue reserving their + ports and SHM budget until expiry. UID matching and mount scoping are + routing/ownership identities, not authentication: host root, privileged + Pods, and processes mounting the parent host directory remain outside this + isolation boundary. Co-located pools must therefore remain inside one + mutually trusted node domain; host firewall controls are required, and + NetworkPolicy does not isolate host-network listeners. - **Runtime consistency:** A pool cannot mix vLLM and SGLang. CacheBackend supplies one server image, chunk size, port tuple, generation, and runtime for the pool. The inference-system owner remains responsible for engine @@ -976,6 +980,9 @@ The final contract is: - [x] Derive one full UID-scoped POSIX SHM name per CacheBackend, pass it explicitly to every NodeLocal server, verify declared and effective live configuration, and replace or un-cover servers missing that identity. +- [x] Mount only `/dev/shm/inference-cache/` as `/dev/shm` in + each NodeLocal server and selected engine; replace or un-cover a server + whose declared hostPath does not match its owner UID. - [x] Derive the endpoint from Downward API node data and render no MP Service or load-balanced ClusterIP. - [x] Keep vLLM and SGLang launch/configuration surfaces separate while sharing @@ -1009,6 +1016,7 @@ Local and repository validation completed on 2026-08-12 PDT: | Legacy production search | Production Go/manifests contain no `LMCacheConnectorV1`, `LMCACHE_REMOTE_URL`, `LMCACHE_REMOTE_SERDE`, `ProtocolLMCache`, `lm://`, or LMCacheServer provider path. Remaining LMCacheServer matches are sample comments explicitly describing its removal. | | Confirmed SHM collision root cause | A focused SJC dev test placed two independent LMCache 0.5.3 standalone Pods on node `10.0.103.182`, with disjoint ports and instance IDs but no `--shm-name`. Both ran as PID 1: A created `/dev/shm/lmcache_l1_pool_1` at inode `14092`; B unlinked that name and recreated inode `14097`; A retained a mapping to deleted inode `14092`. The dedicated namespace was deleted and no control-plane object changed. | | UID-scoped SHM implementation | `git diff --check`, gate-script Python syntax parsing, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check`, and complete `make ci` passed. Tests cover deterministic full-UID naming, distinct UIDs, unsafe/oversized UID rejection, exact server args/annotation, declared and effective startup-gate checks, status exclusion, automatic replacement of an existing server missing the managed SHM identity, and PodLocal regression. Fresh-install could not run locally because no kind node image/cluster is cached and the standard cert-manager bootstrap requires the known-unavailable GitHub path; real envtest API-server admission did run. | +| UID-directory mount hardening | Local validation passed `git diff --check`, `go test ./...`, `make verify-samples` (27 passed, one explicit skip), `make cover-check` at 90.0%, complete `make ci`, and a fresh Kubernetes 1.32 kind install smoke. Tests cover stable and distinct full-UID host paths, exact `DirectoryOrCreate` mounts in server and engine Pods, rejection of the whole host `/dev/shm` and another backend's directory, status exclusion, automatic stale-server replacement, PodLocal regression, current samples, and idempotent re-apply. The live kind node physically created `/dev/shm/inference-cache/` as `root:root 0755`; focused SJC GPU evidence for the pinned root engine/server identities is recorded below. The temporary kind cluster was deleted. Arbitrary non-root runtime compatibility remains outside this validation. | | Focused live SHM remediation | On SJC Kubernetes 1.31.1, two raw LMCache 0.5.3 servers with distinct explicit UID-style names ran together on CPU node `10.0.103.182`: A remained at inode `14166`, while B used inode `14176` and then `14181` after replacement. A's mapping remained named and unchanged throughout. The current controller then created two independent NodeLocal pools on the same node with real CacheBackend UIDs `61a98028-653a-4cfb-83ef-2dc3a9321b50` and `47205c02-6d7d-45aa-bf40-0e1882346309`: their effective names and inodes were respectively `14196` and `14200`; replacing only B moved it to `14207` while A stayed `14196`; deleting and recreating B's engine demand inside idle retention reused B's same server Pod UID and inode `14207`. Both pools reported server/engine coverage `1/1/1/1`. All CPU test resources were deleted. | SJC engine-first GPU validation ran on 2026-08-12 PDT: @@ -1041,7 +1049,8 @@ mechanism and was not supplied by CacheBackend. | Representative common-MP metrics | A focused vLLM TP=1 follow-up used the same pinned vLLM and standalone-server images and checksummed LMCache 0.5.3 wheel. Before traffic, `/metrics` reported L1 usage `0`. A 1,521-token request logged `Stored 1280 tokens`, raised `lmcache_mp_l1_write_chunks_total` to `5`, and raised L1 usage to `15,728,640` bytes. After successful `/reset_prefix_cache`, the identical request logged `Retrieved 1280 tokens in 0.002 seconds`; requested/hit counters became `2560/1280`, and vLLM reported a 42.1% external prefix-cache hit rate. vLLM and SGLang data-plane correctness had already passed separately above; the FastAPI `/metrics` endpoint and counters belong to their common standalone MP server, so this focused follow-up did not repeat the full runtime/node/Redis matrix. | | Post-rebase engine-metrics merge | A focused current-controller test used controller `sha256:a318ea5e96bbcd0ea10f33394beb8fdaa74ce8a93424f6a38f8df75edb4e6889` and subscriber `sha256:ed8a2ad680d248be3e737adf4ba09cd289f6b99909cec580d1cf240d877b3d88`. Live vLLM admission rendered `--hash-scheme=vllm` and the default `http://127.0.0.1:8000/metrics`. A real SGLang 0.5.13.post1 TP=1 Pod instead used its explicitly configured port 8000 and rendered `--hash-scheme=sglang`, `--engine-metrics-url=http://127.0.0.1:8000/metrics`, and `--enable-metrics`. Its endpoint exposed the expected `sglang:token_usage`, `sglang:cache_hit_rate`, `sglang:num_running_reqs`, and `sglang:num_queue_reqs` families; an active request produced `num_running_reqs=1`. Before engine startup the subscriber reported connection failures and `load_signal_stale`; after the endpoint came up it logged `load_signal_recovered`. The authenticated server snapshot then reported `statsReported=true`, `pressure=0.00390625`, one prefix, and a current update timestamp for the SGLang replica. This validates the rebased per-engine profile selection and custom-port plumbing without repeating the already-completed runtime/node/Redis matrix. | | UID-scoped two-pool GPU isolation | A final vLLM TP=1 run placed two CacheBackends and two engines on A100 node `10.0.75.171`, using disjoint ports `15655/39180` and `15656/39181`, 8 GiB L1, chunk size 256, and one GPU/CPU worker per pool. The tested controller was `sha256:0320ba07bae7bf5158ca1120e96c8e31275bf0b2e879a89cde46a48d0f8edc9b`; the server was the pinned standalone digest above; the vLLM digest was `sha256:f72dd35b1efd50fd7646ebce708f173a4040fddf3f2363759c67ad732d912d0a`; and the checksummed wheel carrier was `sha256:81b6767d1435f41832d3494eee47f93d08998cba99f50e9b019d6a7ba7ea1e33`. Both gate checks verified the exact full-UID name in declared and effective `/config`. A stored 1,536 tokens; B's first request for the identical prompt still missed and independently stored 1,536, proving it did not retrieve A's object. After each engine's `/reset_prefix_cache`, each server independently retrieved 1,536 tokens. Metrics for each pool were write/read chunks `6/6`, lookup requested/hit tokens `3072/1536`, L1 usage `18,874,368` bytes, and zero L2 adapters. The servers registered different GPUs (`GPU-4d9375ae-f17c-3721-2417-3af8a961c530` and `GPU-eba663df-3529-f1ea-0da8-6fbe522719d9`) and neither registered the other's worker. Recreating B changed its engine/worker identity but preserved B's server Pod UID and L1; its first request retrieved 1,536 from retained B L1 while A remained unchanged. LMCache did not retain a visible UID-named file in `/dev/shm` during this GPU-worker run, so named-inode lifetime is established by the focused CPU tests above; the GPU test establishes effective-config, endpoint, worker-registration, and behavioral data isolation. | -| Cleanup/control-plane restore | All test objects were removed. The original validation restored from `/private/tmp/inference-cache-phase8-engine-first-sjc-backup-20260812`; the focused metrics run restored from `/private/tmp/inference-cache-phase8-metrics-backup-20260812`; the UID-scoped run restored from `/private/tmp/inference-cache-phase8-shm-backup-20260812`; and the post-rebase metrics run restored from `/private/tmp/inference-cache-phase8-rebase-metrics-backup-20260812`. Semantic comparisons of the CRD spec, ClusterRole rules, ClusterRoleBinding role/subjects, controller and server Deployment specs, and both webhook lists returned no differences after the final run. The original controller digest `sha256:6dcab2344027ef8ac3db2ab22352cdaa77d80202ec11df49dddeeefe08095b18` returned `1/1` Ready, and the test namespace had no remaining workload or CacheBackend. | +| UID-directory GPU isolation | A focused vLLM TP=1 follow-up used controller `sha256:e9c760a3942de447080f9d4373adfef8dd165c4d3ce432311bb9314baecd29bd`, the same pinned standalone/vLLM/wheel artifacts, Kubernetes 1.31.1, A100-SXM4-80GB, driver 550.163.01, and CUDA 12.9 on node `10.0.75.171`. Backend A UID `78346202-561d-4ce4-94dc-081fa7ecbaf5` mounted host `/dev/shm/inference-cache/` as `/dev/shm` in one server and two engines; both engines saw the same tmpfs mount root and inode `262725037`. A1 stored 1,536 of a 1,681-token prompt; fresh A2 retrieved those 1,536 tokens through the same server. A used ports `15755/39280`, 8 GiB L1, chunk size 256, and two GPU workers; status reached servers `1/1` and engines matched/ready/covered `2/2/2`. After A2 was removed, backend B UID `1632dcc4-9c77-40d6-9583-045c4e33be32` started beside A on disjoint ports `15756/39281` and mounted only `/dev/shm/inference-cache/` with distinct inode `262168102`. Both kubelet-created mounts were `root:root 0755`, and the pinned server and engine ran as UID 0 and successfully used them. B's first request for A's identical prompt had zero hits and independently stored 1,536 tokens while every A metric stayed unchanged; after B's `/reset_prefix_cache`, B retrieved its own 1,536 tokens. Final B metrics were write/read chunks `6/6`, requested/hit tokens `3072/1536`, L1 usage `18,874,368` bytes, and zero L2 adapters. Final A/B status was independently `desired=1`, `ready=1`, `matched=1`, `readyEngine=1`, and `covered=1`. The engine Pods did not use host networking or host IPC; each host-networked server declared its exact host-port pair and full UID `--shm-name`. | +| Cleanup/control-plane restore | All test objects were removed. The original validation restored from `/private/tmp/inference-cache-phase8-engine-first-sjc-backup-20260812`; the focused metrics run restored from `/private/tmp/inference-cache-phase8-metrics-backup-20260812`; the UID-scoped run restored from `/private/tmp/inference-cache-phase8-shm-backup-20260812`; the post-rebase metrics run restored from `/private/tmp/inference-cache-phase8-rebase-metrics-backup-20260812`; and the UID-directory run restored from `/private/tmp/inference-cache-phase8-uid-dir-backup-20260812`. The final run also removed only its two exact UID host directories after confirming that deleted engine/server Pods had left CUDA, torch, and semaphore files behind. Semantic comparisons of the CRD spec, ClusterRole rules, ClusterRoleBinding role/subjects, controller Deployment spec, and both webhook lists returned no differences after the final run. The original controller digest `sha256:6dcab2344027ef8ac3db2ab22352cdaa77d80202ec11df49dddeeefe08095b18` returned `1/1` Ready, and the test namespace had no remaining workload or CacheBackend. | - [x] Zero servers before an engine is scheduled; one healthy same-node server after the first vLLM or SGLang TP=1 engine is placed. @@ -1077,9 +1086,13 @@ mechanism and was not supplied by CacheBackend. - [x] Cross-`CacheBackend` sharing remains rejected by the name+UID demand filter. - [x] Required vLLM and SGLang functional matrix evidence is supplemented by before/after metrics from their common standalone MP-server data path. -- [x] UID-scoped NodeLocal SHM isolation passes focused live-node and GPU - validation; missing or mismatched SHM identity never counts as Ready or +- [x] UID-scoped SHM object-name isolation passes focused live-node and GPU + validation; missing or mismatched name identity never counts as Ready or covered and never admits an engine. +- [x] UID-directory mounts pass focused SJC GPU validation for one pool, two + same-node engines, and two co-located CacheBackends; the pinned root + engine/server identities successfully use the kubelet-created + `root:root 0755` UID directories. ## Required GPU validation matrix @@ -1126,10 +1139,14 @@ The migration is complete only when all of the following are true: - [x] NodeLocal, if enabled, guarantees same-node server selection and accurate engine coverage; otherwise it remains rejected rather than partially accepted. +- [x] The UID-directory NodeLocal data path passes focused GPU validation, + including LMCache access through the kubelet-created directory and + isolation between two co-located CacheBackends. -Phase 8 is complete, including focused UID-scoped SHM validation, and is the -final phase of this migration. The future capability profiles below are -independent backlog items rather than additional migration phases. +Phase 8 production behavior and its UID-scoped object-name and directory-mount +validation are complete. This remains the final phase of the migration; the +future capability profiles below are independent backlog items rather than +additional phases. ## Known limitations and future work @@ -1142,19 +1159,30 @@ immutable artifacts. The numbered items below are the future-work backlog. ### 1. NodeLocal hostile-process isolation and aggregate SHM capacity -Phase 8 owns the accidental-collision fix and its correctness validation: every -NodeLocal pool now uses a deterministic full-UID `--shm-name`, and startup/status -verify that exact identity. This future item covers the stronger security and -capacity guarantees that unique names cannot provide. +Phase 8 owns the accidental-collision fix: every NodeLocal pool uses a +deterministic full-UID `--shm-name` and mounts only its full-UID host directory; +startup/status verify that exact identity. This future item covers stronger +security and capacity guarantees that mount scoping cannot provide. -- [ ] Define the hostile-process boundary. A unique name does not stop a - same-node process with host `/dev/shm` access and compatible Unix - credentials from deliberately opening or unlinking another pool. Decide - whether production support requires distinct Unix identities, isolated - SHM backing, admission-enforced node separation, or a combination. +- [ ] Define the hostile-process boundary. A UID-directory mount does not stop + host root, a privileged Pod, or another process that independently mounts + the parent host `/dev/shm` from deliberately opening or unlinking another + pool. Decide whether production support requires distinct Unix + identities, admission-enforced node separation, or a combination. - [ ] Account for aggregate host `/dev/shm` capacity across co-located pools and expose actionable admission/Pending/status behavior before publishing a supported multi-pool capacity envelope. +- [ ] Define ownership-verified reclamation for an idle/deleted pool's UID + directory. The focused GPU run found CUDA, torch, and semaphore files + still present after every engine and server Pod had exited. This SHM + lifetime behavior pre-dates UID directories: under the former whole-host + mount the same classes of objects shared the unowned `/dev/shm` root and + could not be attributed or safely reclaimed. UID scoping makes that + existing lifecycle problem ownership-visible; it adds only the directory + entry itself. Without safe last-user cleanup, the objects and their tmpfs + pages can remain until explicit node cleanup or reboot. Cleanup must prove + that no selected engine or server still uses the directory and must never + traverse or delete another CacheBackend UID. - [ ] Validate the selected tenant boundary with unauthorized open/unlink tests; until then, multiple pools on one node are supported only inside one mutually trusted node domain. diff --git a/docs/design/lmcache-server-persistence.md b/docs/design/lmcache-server-persistence.md index 91f1658d..4656c531 100644 --- a/docs/design/lmcache-server-persistence.md +++ b/docs/design/lmcache-server-persistence.md @@ -38,8 +38,10 @@ PVC**: mount storage nothing writes KV to. 2. **LMCache's only on-server local-disk path is node-local.** Its MP-mode (the L2 NIXL POSIX backend writing to a `file_path`) requires `hostNetwork` and a - shared host `/dev/shm`, where the control socket is ZMQ-only and KV bytes move - over CUDA-IPC or POSIX shared memory. A server reachable only through a + shared node tmpfs path. NodeLocal mounts the CacheBackend UID's + `/dev/shm/inference-cache/` directory as `/dev/shm` in its server and + engines; the control socket is ZMQ-only and KV bytes move over CUDA-IPC or + POSIX shared memory. A server reachable only through a ClusterIP Service therefore has **no data plane** in that mode. The current implementation creates one directly scheduled server Pod per active engine node and CacheBackend; multiple pools on one node require disjoint host ports. diff --git a/docs/quickstart.md b/docs/quickstart.md index 38549ec4..d0eb0dbd 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -55,8 +55,10 @@ focused NodeLocal samples for NodeLocal is an advanced host-bound topology. The inference system schedules each engine without CacheBackend changing its placement; the controller then creates one shared server Pod on every node that actually has an active -selected engine. The server mounts host `/dev/shm` and declares MP and HTTP host -ports. Engines are held in an init gate until their own node's server reports +selected engine. The server and engine mount only the backend's +`/dev/shm/inference-cache/` host directory as `/dev/shm` and declare MP +and HTTP host ports. Engines are held in an init gate until their own node's +server reports the exact CacheBackend name/UID/generation and healthy config. Co-schedule only mutually trusted engines and enforce host-port access with node firewalls; hostNetwork bypasses Kubernetes NetworkPolicy. L1 capacity is per node, and diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 776e72a3..b5845b22 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -368,6 +368,10 @@ grep -Fq 'EXPECTED_SHM_NAME' "$node_pod_json" || fail "NodeLocal startup gate do grep -Fq 'INFERENCECACHE_NODE_IP' "$node_pod_json" || fail "NodeLocal hostIP Downward API was not injected" grep -Fq 'status.hostIP' "$node_pod_json" || fail "NodeLocal endpoint is not derived from status.hostIP" grep -Fq 'kubernetes.io/os' "$node_pod_json" || fail "inference-owned nodeSelector was not preserved" +node_local_backend_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend node-local -o jsonpath='{.metadata.uid}')" +node_local_shm_path="/dev/shm/inference-cache/${node_local_backend_uid}" +grep -Fq "$node_local_shm_path" "$node_pod_json" || fail "NodeLocal engine does not mount its UID-scoped host SHM directory" +grep -Fq 'DirectoryOrCreate' "$node_pod_json" || fail "NodeLocal engine UID-scoped SHM hostPath is not DirectoryOrCreate" if grep -Fq 'podAffinity' "$node_pod_json"; then fail "NodeLocal injection unexpectedly added server-first PodAffinity" fi @@ -394,9 +398,10 @@ kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o json >"$LOG_DIR/node-loc node_local_host_ipc="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.hostIPC}')" [ -z "$node_local_host_ipc" ] || [ "$node_local_host_ipc" = "false" ] \ || fail "NodeLocal server unexpectedly uses hostIPC" -[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.volumes[0].hostPath.path}')" = "/dev/shm" ] \ - || fail "NodeLocal server does not mount host /dev/shm" -node_local_backend_uid="$(kubectl -n "$SMOKE_NAMESPACE" get cachebackend node-local -o jsonpath='{.metadata.uid}')" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.volumes[0].hostPath.path}')" = "$node_local_shm_path" ] \ + || fail "NodeLocal server does not mount its UID-scoped host SHM directory" +[ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.spec.volumes[0].hostPath.type}')" = "DirectoryOrCreate" ] \ + || fail "NodeLocal server UID-scoped SHM hostPath is not DirectoryOrCreate" node_local_shm_name="lmcache_l1_pool_inferencecache_${node_local_backend_uid}" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{.metadata.annotations.inferencecache\.io/node-local-shm-name}')" = "$node_local_shm_name" ] \ || fail "NodeLocal server does not carry its UID-scoped shared-memory identity" diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go index 4caa0564..8fb0ea6d 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal.go @@ -30,6 +30,7 @@ const ( lmCacheNodeLocalConfigFilePath = lmCacheNodeLocalConfigMountPath + "/client.yaml" lmCacheNodeIPEnv = "INFERENCECACHE_NODE_IP" lmCacheNodeLocalShmNamePrefix = "lmcache_l1_pool_inferencecache_" + lmCacheNodeLocalShmHostRoot = "/dev/shm/inference-cache" posixShmNameMaxLength = 255 ) @@ -68,6 +69,10 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding if err != nil { return nil, err } + shmHostPath, err := NodeLocalServerShmHostPath(cache) + if err != nil { + return nil, err + } args := []string{ "server", "--instance-id", identity, @@ -128,7 +133,7 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding enginebinding.AnnotationNodeLocalTargetNode: nodeName, enginebinding.AnnotationNodeLocalShmName: shmName, } - pathType := corev1.HostPathDirectory + pathType := corev1.HostPathDirectoryOrCreate noToken := false enableServiceLinks := false grace := int64(30) @@ -177,7 +182,7 @@ func RenderLMCacheNodeLocalServerPod(cache *cachev1alpha1.CacheBackend, binding Volumes: []corev1.Volume{{ Name: lmCacheNodeLocalShmVolumeName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: lmCacheMPShmMountPath, Type: &pathType, + Path: shmHostPath, Type: &pathType, }}, }}, } @@ -231,6 +236,17 @@ func NodeLocalServerShmName(cache *cachev1alpha1.CacheBackend) (string, error) { return name, nil } +// NodeLocalServerShmHostPath returns the host tmpfs directory mounted as +// /dev/shm by one CacheBackend's NodeLocal servers and engines. Mounting only +// the UID directory keeps normally behaving co-located pools out of each +// other's POSIX SHM namespace while retaining the node-local CUDA IPC path. +func NodeLocalServerShmHostPath(cache *cachev1alpha1.CacheBackend) (string, error) { + if _, err := NodeLocalServerShmName(cache); err != nil { + return "", err + } + return lmCacheNodeLocalShmHostRoot + "/" + string(cache.UID), nil +} + func exactNodeAffinity(nodeName string) *corev1.Affinity { return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ @@ -306,9 +322,9 @@ func lmCacheMPHTTPProbeForPort(portName string, period, failures int32) *corev1. return probe } -// renderLMCacheNodeLocalEngine injects the shared host /dev/shm mount and a -// startup gate. It deliberately leaves every engine placement field unchanged; -// the controller follows the engine onto its scheduled node. +// renderLMCacheNodeLocalEngine injects the backend-UID-scoped host SHM mount +// and a startup gate. It deliberately leaves every engine placement field +// unchanged; the controller follows the engine onto its scheduled node. func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName string, cache *cachev1alpha1.CacheBackend, writeClientConfig bool) (string, error) { if pod == nil { return "", fmt.Errorf("render LMCache NodeLocal engine: pod spec is nil") @@ -329,6 +345,10 @@ func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName strin if err != nil { return "", err } + shmHostPath, err := NodeLocalServerShmHostPath(cache) + if err != nil { + return "", err + } work := pod.DeepCopy() engine := &work.Containers[engineIndex] owned := lmCacheNodeLocalWireIsOurs(pod) @@ -347,16 +367,16 @@ func renderLMCacheNodeLocalEngine(pod *corev1.PodSpec, engineContainerName strin shmMount := corev1.VolumeMount{Name: lmCacheNodeLocalShmVolumeName, MountPath: lmCacheMPShmMountPath} if existing := mountAtPath(engine.VolumeMounts, lmCacheMPShmMountPath); existing != nil { - if err := checkNodeLocalHostShm(work.Volumes, *existing); err != nil { + if err := checkNodeLocalHostShm(work.Volumes, *existing, shmHostPath); err != nil { return "", err } shmMount.Name = existing.Name } else { - pathType := corev1.HostPathDirectory + pathType := corev1.HostPathDirectoryOrCreate work.Volumes, err = adoptVolume(work.Volumes, corev1.Volume{ Name: lmCacheNodeLocalShmVolumeName, VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: lmCacheMPShmMountPath, Type: &pathType, + Path: shmHostPath, Type: &pathType, }}, }, owned) if err != nil { @@ -467,7 +487,7 @@ while True: } } -func checkNodeLocalHostShm(volumes []corev1.Volume, mount corev1.VolumeMount) error { +func checkNodeLocalHostShm(volumes []corev1.Volume, mount corev1.VolumeMount, wantHostPath string) error { if mount.ReadOnly || mount.SubPath != "" || mount.SubPathExpr != "" { return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm must be a writable whole-volume hostPath mount") } @@ -476,8 +496,8 @@ func checkNodeLocalHostShm(volumes []corev1.Volume, mount corev1.VolumeMount) er continue } hostPath := volumes[i].HostPath - if hostPath == nil || hostPath.Path != lmCacheMPShmMountPath { - return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm must use hostPath /dev/shm for cross-Pod CUDA IPC") + if hostPath == nil || hostPath.Path != wantHostPath || hostPath.Type == nil || *hostPath.Type != corev1.HostPathDirectoryOrCreate { + return fmt.Errorf("render LMCache NodeLocal engine: /dev/shm must use UID-scoped hostPath %q with type DirectoryOrCreate for cross-Pod CUDA IPC", wantHostPath) } return nil } diff --git a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go index 9041fc7f..211ede0b 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_nodelocal_test.go @@ -86,8 +86,10 @@ func TestRenderLMCacheNodeLocalServerPod(t *testing.T) { !reflect.DeepEqual(required.NodeSelectorTerms[0].MatchFields[0].Values, []string{"gpu-node-a"}) { t.Fatalf("exact-node scheduler affinity = %+v", required) } - if len(pod.Volumes) != 1 || pod.Volumes[0].HostPath == nil || pod.Volumes[0].HostPath.Path != "/dev/shm" { - t.Fatalf("volumes = %+v, want host /dev/shm", pod.Volumes) + wantShmPath := "/dev/shm/inference-cache/11111111-2222-3333-4444-555555555555" + if len(pod.Volumes) != 1 || pod.Volumes[0].HostPath == nil || pod.Volumes[0].HostPath.Path != wantShmPath || + pod.Volumes[0].HostPath.Type == nil || *pod.Volumes[0].HostPath.Type != corev1.HostPathDirectoryOrCreate { + t.Fatalf("volumes = %+v, want UID-scoped hostPath %q", pod.Volumes, wantShmPath) } if len(pod.Containers) != 1 { t.Fatalf("containers = %d, want one", len(pod.Containers)) @@ -210,6 +212,25 @@ func TestNodeLocalServerShmNameIsStableAndUIDScoped(t *testing.T) { } } +func TestNodeLocalServerShmHostPathIsStableAndUIDScoped(t *testing.T) { + cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) + want := "/dev/shm/inference-cache/11111111-2222-3333-4444-555555555555" + got, err := NodeLocalServerShmHostPath(cache) + if err != nil || got != want { + t.Fatalf("NodeLocalServerShmHostPath = %q, %v; want %q", got, err, want) + } + cache.Generation++ + stable, err := NodeLocalServerShmHostPath(cache) + if err != nil || stable != want { + t.Fatalf("same-UID replacement host path = %q, %v; want %q", stable, err, want) + } + cache.UID = types.UID("aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee") + distinct, err := NodeLocalServerShmHostPath(cache) + if err != nil || distinct == want { + t.Fatalf("different-UID host path = %q, %v; must differ from %q", distinct, err, want) + } +} + func TestNodeLocalServerShmNameRejectsUnsafeUID(t *testing.T) { cache := newNodeLocalBackend(cachev1alpha1.CacheBackendRuntimeVLLM) cache.UID = types.UID("unsafe/uid") @@ -253,9 +274,21 @@ func TestVLLMNodeLocalEngineInjection(t *testing.T) { if !strings.Contains(joined, `tcp://$(INFERENCECACHE_NODE_IP)`) || !strings.Contains(joined, `"lmcache.mp.port":"6555"`) { t.Fatalf("vLLM args do not carry node-derived endpoint: %s", joined) } - if mountAtPath(engine.VolumeMounts, "/dev/shm") == nil { + shmMount := mountAtPath(engine.VolumeMounts, "/dev/shm") + if shmMount == nil { t.Fatalf("engine mounts = %+v, want host /dev/shm", engine.VolumeMounts) } + var shmVolume *corev1.Volume + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name == shmMount.Name { + shmVolume = &pod.Spec.Volumes[i] + break + } + } + if shmVolume == nil || shmVolume.HostPath == nil || shmVolume.HostPath.Path != "/dev/shm/inference-cache/11111111-2222-3333-4444-555555555555" || + shmVolume.HostPath.Type == nil || *shmVolume.HostPath.Type != corev1.HostPathDirectoryOrCreate { + t.Fatalf("engine SHM volume = %+v, want backend UID directory", shmVolume) + } gate := pod.Spec.InitContainers[0] if !strings.Contains(gate.Args[0], "/config") || !strings.Contains(gate.Args[0], "EXPECTED_INSTANCE_ID") || !strings.Contains(gate.Args[0], `memory.get("shm_name")`) { @@ -383,7 +416,16 @@ func TestNodeLocalEngineInjectionRejectsReservedWireCollisionsAtomically(t *test pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"}) }, - want: "must use hostPath", + want: "UID-scoped hostPath", + }, + { + name: "whole host shared memory", + mutate: func(pod *corev1.PodSpec) { + pathType := corev1.HostPathDirectory + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{Path: "/dev/shm", Type: &pathType}}}) + pod.Containers[0].VolumeMounts = append(pod.Containers[0].VolumeMounts, corev1.VolumeMount{Name: "shm", MountPath: "/dev/shm"}) + }, + want: "UID-scoped hostPath", }, { name: "missing shared memory volume", diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index a7c80bf0..892780d1 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -224,6 +224,11 @@ func (r *CacheBackendReconciler) refreshLMCacheNodeLocalConnectorStatus(ctx cont log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: shared-memory identity is invalid", "error", shmNameErr.Error()) return } + wantShmHostPath, shmPathErr := builtinruntime.NodeLocalServerShmHostPath(backend) + if shmPathErr != nil { + log.FromContext(ctx).V(1).Info("LMCache NodeLocal status refresh skipped: shared-memory host path is invalid", "error", shmPathErr.Error()) + return + } readyByNode := map[string]int32{} conflictByNode := map[string]bool{} for i := range servers.Items { @@ -235,7 +240,7 @@ func (r *CacheBackendReconciler) refreshLMCacheNodeLocalConnectorStatus(ctx cont if annotations[enginebinding.AnnotationNodeLocalOwner] != wantOwner || annotations[enginebinding.AnnotationNodeLocalOwnerUID] != wantUID || annotations[enginebinding.AnnotationNodeLocalGeneration] != wantGeneration || - !nodeLocalServerHasShmIdentity(pod, wantShmName) { + !nodeLocalServerHasShmIdentity(pod, wantShmName, wantShmHostPath) { continue } targetNode := annotations[enginebinding.AnnotationNodeLocalTargetNode] diff --git a/internal/controller/cachebackend_lmcache_mp_status_test.go b/internal/controller/cachebackend_lmcache_mp_status_test.go index 340ba012..bafee175 100644 --- a/internal/controller/cachebackend_lmcache_mp_status_test.go +++ b/internal/controller/cachebackend_lmcache_mp_status_test.go @@ -32,8 +32,19 @@ func setNodeLocalShmIdentity(t *testing.T, backend *cachev1alpha1.CacheBackend, if pod.Annotations == nil { pod.Annotations = map[string]string{} } + path, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + pathType := corev1.HostPathDirectoryOrCreate pod.Annotations[enginebinding.AnnotationNodeLocalShmName] = name - pod.Spec.Containers = []corev1.Container{{Name: lmCacheMPServerStatusContainerName, Args: []string{"server", "--shm-name", name}}} + pod.Spec.Containers = []corev1.Container{{ + Name: lmCacheMPServerStatusContainerName, Args: []string{"server", "--shm-name", name}, + VolumeMounts: []corev1.VolumeMount{{Name: "shm", MountPath: "/dev/shm"}}, + }} + pod.Spec.Volumes = []corev1.Volume{{Name: "shm", VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: path, Type: &pathType, + }}}} } func typedMPStatusBackend() *cachev1alpha1.CacheBackend { @@ -325,6 +336,13 @@ func TestRefreshLMCacheNodeLocalConnectorStatusFailureModes(t *testing.T) { }, wantCondition: reasonNodeLocalPoolPending, wantCoverage: reasonMPServersNotReady, wantReadyServers: 0, }, + { + name: "server mounts another shared-memory directory", + mutate: func(_ *cachev1alpha1.CacheBackend, servers *[]*corev1.Pod, _ *corev1.Pod) { + (*servers)[0].Spec.Volumes[0].HostPath.Path = "/dev/shm/inference-cache/another-backend-uid" + }, + wantCondition: reasonNodeLocalPoolPending, wantCoverage: reasonMPServersNotReady, wantReadyServers: 0, + }, { name: "ambiguous ready servers on one node", mutate: func(_ *cachev1alpha1.CacheBackend, servers *[]*corev1.Pod, _ *corev1.Pod) { diff --git a/internal/controller/cachebackend_lmcache_nodelocal.go b/internal/controller/cachebackend_lmcache_nodelocal.go index aab6ddb3..ae8bce16 100644 --- a/internal/controller/cachebackend_lmcache_nodelocal.go +++ b/internal/controller/cachebackend_lmcache_nodelocal.go @@ -52,6 +52,10 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context if err != nil { return err } + wantShmHostPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + return err + } var servers corev1.PodList if err := r.Client.List(ctx, &servers, @@ -76,7 +80,7 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context current := pod.Annotations[enginebinding.AnnotationNodeLocalOwnerUID] == string(backend.UID) && pod.Annotations[enginebinding.AnnotationNodeLocalGeneration] == wantGeneration && pod.Name == builtinruntime.NodeLocalServerPodName(backend.Name, targetNode) && - nodeLocalServerHasShmIdentity(pod, wantShmName) + nodeLocalServerHasShmIdentity(pod, wantShmName, wantShmHostPath) if !current { if pod.DeletionTimestamp == nil { if err := r.Client.Delete(ctx, pod); err != nil && !apierrors.IsNotFound(err) { @@ -165,17 +169,41 @@ func (r *CacheBackendReconciler) reconcileLMCacheNodeLocalServerPods(ctx context return nil } -func nodeLocalServerHasShmIdentity(pod *corev1.Pod, want string) bool { - if pod == nil || want == "" || pod.Annotations[enginebinding.AnnotationNodeLocalShmName] != want { +func nodeLocalServerHasShmIdentity(pod *corev1.Pod, wantName, wantHostPath string) bool { + if pod == nil || wantName == "" || wantHostPath == "" || pod.Annotations[enginebinding.AnnotationNodeLocalShmName] != wantName { return false } for i := range pod.Spec.Containers { if pod.Spec.Containers[i].Name != lmCacheMPServerStatusContainerName { continue } - args := pod.Spec.Containers[i].Args + container := &pod.Spec.Containers[i] + var shmVolumeName string + for j := range container.VolumeMounts { + mount := &container.VolumeMounts[j] + if mount.MountPath == "/dev/shm" && !mount.ReadOnly && mount.SubPath == "" && mount.SubPathExpr == "" { + shmVolumeName = mount.Name + break + } + } + if shmVolumeName == "" { + return false + } + validHostPath := false + for j := range pod.Spec.Volumes { + volume := &pod.Spec.Volumes[j] + if volume.Name == shmVolumeName && volume.HostPath != nil && volume.HostPath.Path == wantHostPath && + volume.HostPath.Type != nil && *volume.HostPath.Type == corev1.HostPathDirectoryOrCreate { + validHostPath = true + break + } + } + if !validHostPath { + return false + } + args := container.Args for j := 0; j+1 < len(args); j++ { - if args[j] == "--shm-name" && args[j+1] == want { + if args[j] == "--shm-name" && args[j+1] == wantName { return true } } diff --git a/internal/controller/cachebackend_mp_lifecycle_test.go b/internal/controller/cachebackend_mp_lifecycle_test.go index 7d2a287b..07dae13a 100644 --- a/internal/controller/cachebackend_mp_lifecycle_test.go +++ b/internal/controller/cachebackend_mp_lifecycle_test.go @@ -329,11 +329,55 @@ func TestReconcileNodeLocalReplacesServerMissingUIDScopedShmIdentity(t *testing. if err != nil { t.Fatal(err) } - if !nodeLocalServerHasShmIdentity(&replaced, want) { + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + if !nodeLocalServerHasShmIdentity(&replaced, want, wantPath) { t.Fatalf("replacement server lacks UID-scoped shm identity: annotations=%v args=%v", replaced.Annotations, replaced.Spec.Containers[0].Args) } } +func TestReconcileNodeLocalReplacesServerWithWrongUIDScopedShmDirectory(t *testing.T) { + backend := nodeLocalBackend("node-cache", "ns1") + engine := nodeLocalEngine(backend, "engine-a", "node-a") + reconciler := newReconciler(newScheme(t), backend, engine) + reconcile(t, reconciler, backend.Name, backend.Namespace) + + key := types.NamespacedName{Name: builtinruntime.NodeLocalServerPodName(backend.Name, "node-a"), Namespace: backend.Namespace} + var old corev1.Pod + if err := reconciler.Get(context.Background(), key, &old); err != nil { + t.Fatalf("get original server Pod: %v", err) + } + if err := reconciler.Delete(context.Background(), &old); err != nil { + t.Fatalf("delete original server Pod: %v", err) + } + old.ResourceVersion = "" + old.UID = "" + old.CreationTimestamp = metav1.Time{} + old.Spec.Volumes[0].HostPath.Path = "/dev/shm/inference-cache/foreign-cachebackend-uid" + if err := reconciler.Create(context.Background(), &old); err != nil { + t.Fatalf("create server Pod with foreign SHM directory: %v", err) + } + + reconcile(t, reconciler, backend.Name, backend.Namespace) + var replaced corev1.Pod + if err := reconciler.Get(context.Background(), key, &replaced); err != nil { + t.Fatalf("get replacement server Pod: %v", err) + } + wantName, err := builtinruntime.NodeLocalServerShmName(backend) + if err != nil { + t.Fatal(err) + } + wantPath, err := builtinruntime.NodeLocalServerShmHostPath(backend) + if err != nil { + t.Fatal(err) + } + if !nodeLocalServerHasShmIdentity(&replaced, wantName, wantPath) { + t.Fatalf("replacement server lacks UID-scoped SHM directory: volumes=%v", replaced.Spec.Volumes) + } +} + func TestCleanupNodeLocalPreservesUnownedServerPod(t *testing.T) { backend := nodeLocalBackend("node-cache", "ns1") pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foreign", Namespace: backend.Namespace, Labels: map[string]string{ diff --git a/internal/controller/cachebackend_nodelocal_integration_test.go b/internal/controller/cachebackend_nodelocal_integration_test.go index a067d220..9863d173 100644 --- a/internal/controller/cachebackend_nodelocal_integration_test.go +++ b/internal/controller/cachebackend_nodelocal_integration_test.go @@ -69,7 +69,11 @@ func TestIntegrationCacheBackendNodeLocalServerPod(t *testing.T) { if err != nil { t.Fatal(err) } - if !nodeLocalServerHasShmIdentity(&server, wantShmName) { + wantShmPath, err := builtinruntime.NodeLocalServerShmHostPath(&live) + if err != nil { + t.Fatal(err) + } + if !nodeLocalServerHasShmIdentity(&server, wantShmName, wantShmPath) { t.Fatalf("server Pod lacks UID-scoped shm identity: annotations=%v args=%v", server.Annotations, server.Spec.Containers[0].Args) } From 9ae86e564458b20b5200ca34db1b854476174fb1 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 12 Aug 2026 23:13:56 -0700 Subject: [PATCH 11/13] Fix PR smoke and patch coverage Signed-off-by: Yue Sun --- .../scripts/default_install_smoke.sh | 2 +- .../runtime/lmcache_mp_renderer_test.go | 60 +++++++++++++ .../builtin/runtime/sglang_lmcache_test.go | 90 +++++++++++++++++++ .../builtin/runtime/vllm_lmcache_mp_test.go | 88 ++++++++++++++++++ .../builtin/storage/effective_config_test.go | 23 +++++ .../controller/cachebackend_reconciler.go | 26 +++--- pkg/adapters/backend/backend_test.go | 21 +++++ 7 files changed, 294 insertions(+), 16 deletions(-) create mode 100644 internal/adapters/builtin/storage/effective_config_test.go diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index b5845b22..8f9a5110 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -407,7 +407,7 @@ node_local_shm_name="lmcache_l1_pool_inferencecache_${node_local_backend_uid}" || fail "NodeLocal server does not carry its UID-scoped shared-memory identity" node_local_shm_arg="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" \ -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].args[*]}{@}{"\n"}{end}' | \ - awk 'previous == "--shm-name" { print; exit } { previous = $0 }')" + awk 'previous == "--shm-name" && value == "" { value = $0 } { previous = $0 } END { print value }')" [ "$node_local_shm_arg" = "$node_local_shm_name" ] \ || fail "NodeLocal server does not pass its UID-scoped --shm-name: $node_local_shm_arg" node_local_ports="$(kubectl -n "$SMOKE_NAMESPACE" get pod "$server_name" -o jsonpath='{range .spec.containers[?(@.name=="lmcache-mp-server")].ports[*]}{.containerPort}:{.hostPort}{" "}{end}')" diff --git a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go index 8aa0e989..c7d0103a 100644 --- a/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go +++ b/internal/adapters/builtin/runtime/lmcache_mp_renderer_test.go @@ -194,6 +194,29 @@ func TestRenderLMCachePodLocalServerCollisionIsAtomic(t *testing.T) { Volumes: []corev1.Volume{{Name: lmCacheMPConfigVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "foreign"}}}}, }, }, + { + name: "foreign config mount", + pod: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine", VolumeMounts: []corev1.VolumeMount{{Name: "foreign-config", MountPath: lmCacheMPConfigMountPath}}}}, + Volumes: []corev1.Volume{{Name: "foreign-config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{}}}}, + }, + }, + { + name: "read-only shm mount", + pod: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine", VolumeMounts: []corev1.VolumeMount{{Name: "engine-shm", MountPath: lmCacheMPShmMountPath, ReadOnly: true}}}}, + Volumes: []corev1.Volume{{Name: "engine-shm", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, SizeLimit: func() *resource.Quantity { q := resource.MustParse("6Gi"); return &q }(), + }}}}, + }, + }, + { + name: "foreign reserved shm volume", + pod: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "engine"}}, + Volumes: []corev1.Volume{{Name: lmCacheMPShmVolumeName, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "foreign"}}}}, + }, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -208,6 +231,43 @@ func TestRenderLMCachePodLocalServerCollisionIsAtomic(t *testing.T) { } } +func TestRenderLMCachePodLocalServerRejectsInvalidInputs(t *testing.T) { + validPod := func() *corev1.PodSpec { + return &corev1.PodSpec{Containers: []corev1.Container{{Name: "engine"}}} + } + tests := []struct { + name string + pod *corev1.PodSpec + cfg lmCacheMPServerConfig + wantErr string + }{ + {name: "nil pod", cfg: testLMCacheMPConfig(), wantErr: "pod spec is nil"}, + {name: "ambiguous engine", pod: &corev1.PodSpec{Containers: []corev1.Container{{Name: "worker"}, {Name: "sidecar"}}}, cfg: testLMCacheMPConfig(), wantErr: "none is named \"engine\""}, + {name: "invalid config", pod: validPod(), cfg: lmCacheMPServerConfig{}, wantErr: "image is empty"}, + {name: "unsupported binding", pod: validPod(), cfg: func() lmCacheMPServerConfig { + cfg := testLMCacheMPConfig() + cfg.Binding = &backendadapter.Binding{Protocol: backendadapter.Protocol("grpc")} + return cfg + }(), wantErr: "unsupported remote protocol"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := renderLMCachePodLocalServer(tc.pod, "engine", tc.cfg) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("renderLMCachePodLocalServer error = %v, want substring %q", err, tc.wantErr) + } + }) + } + + budget := resource.MustParse("5Gi") + if err := checkLMCacheMPShmBudget(nil, corev1.VolumeMount{Name: "missing"}, budget); err == nil || !strings.Contains(err.Error(), "missing volume") { + t.Fatalf("checkLMCacheMPShmBudget error = %v, want missing volume", err) + } + if _, err := lmCacheMPServerContainer(lmCacheMPServerConfig{}, "", nil, corev1.VolumeMount{}); err == nil || !strings.Contains(err.Error(), "greater than zero") { + t.Fatalf("lmCacheMPServerContainer error = %v, want invalid capacity", err) + } +} + func TestRenderLMCachePodLocalServerReusesWritableShm(t *testing.T) { pod := &corev1.PodSpec{ Containers: []corev1.Container{{ diff --git a/internal/adapters/builtin/runtime/sglang_lmcache_test.go b/internal/adapters/builtin/runtime/sglang_lmcache_test.go index 2bf38ac2..1b7d4792 100644 --- a/internal/adapters/builtin/runtime/sglang_lmcache_test.go +++ b/internal/adapters/builtin/runtime/sglang_lmcache_test.go @@ -160,6 +160,18 @@ func TestSGLangInjectsMetricsOnlyWhenSubscriberWillAttach(t *testing.T) { } } +func TestSGLangMetricsInjectionRequiresEngineContainer(t *testing.T) { + cache := observedTypedSGLangBackend("gemma") + err := ensureSGLangMetricsForSubscriber( + &corev1.PodSpec{Containers: []corev1.Container{{Name: "worker"}, {Name: "sidecar"}}}, + cache, + SubscriberConfig{Image: "subscriber:pinned"}, + ) + if err == nil || !strings.Contains(err.Error(), SGLangEngineContainerName) { + t.Fatalf("ensureSGLangMetricsForSubscriber error = %v, want missing engine container", err) + } +} + func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) tests := []struct { @@ -196,6 +208,84 @@ func TestSGLangValidateTypedMPEnginePodPageSize(t *testing.T) { } } +func TestSGLangValidateRejectsIncompleteTopology(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + validPod := &corev1.Pod{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName, Args: []string{"--page-size", "64"}}}}} + tests := []struct { + name string + pod *corev1.Pod + cache *cachev1alpha1.CacheBackend + wantErr string + }{ + {name: "nil pod", cache: typedSGLangBackend(), wantErr: "pod is nil"}, + {name: "nil cache", pod: validPod, wantErr: "configuration is missing"}, + {name: "missing LMCache", pod: validPod, cache: &cachev1alpha1.CacheBackend{}, wantErr: "configuration is missing"}, + {name: "missing PodLocal server", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "PodLocal server configuration is missing"}, + {name: "missing NodeLocal server", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "NodeLocal server configuration is missing"}, + {name: "unsupported topology", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.Topology = "Remote" + return cache + }(), wantErr: "topology \"Remote\" is not implemented"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := adapter.ValidateMPEnginePod(tc.pod, tc.cache) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateMPEnginePod error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + +func TestSGLangInjectRejectsInvalidInputs(t *testing.T) { + adapter := NewSGLangLMCacheAdapter(SubscriberConfig{}) + validPod := corev1.PodSpec{Containers: []corev1.Container{{Name: SGLangEngineContainerName}}} + tests := []struct { + name string + pod *corev1.PodSpec + cache *cachev1alpha1.CacheBackend + wantErr string + }{ + {name: "nil pod", cache: typedSGLangBackend(), wantErr: "pod is nil"}, + {name: "nil cache", pod: &validPod, wantErr: "cache is nil"}, + {name: "missing LMCache", pod: &validPod, cache: &cachev1alpha1.CacheBackend{}, wantErr: "typed server configuration is required"}, + {name: "missing PodLocal server", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "typed PodLocal server configuration is required"}, + {name: "missing NodeLocal server", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "typed NodeLocal server configuration is required"}, + {name: "unsupported topology", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := typedSGLangBackend() + cache.Spec.LMCache.Topology = "Remote" + return cache + }(), wantErr: "topology \"Remote\" is not implemented"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := adapter.InjectEngineConfig(tc.pod, nil, tc.cache) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("InjectEngineConfig error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + func TestSGLangInjectRouterConfigIsNoop(t *testing.T) { a := NewSGLangLMCacheAdapter(SubscriberConfig{}) cb := typedSGLangBackend() diff --git a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go index 02ed4a79..2b67c6ee 100644 --- a/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go +++ b/internal/adapters/builtin/runtime/vllm_lmcache_mp_test.go @@ -152,10 +152,13 @@ func TestVLLMLMCacheMPValidateEngineParallelism(t *testing.T) { {name: "TP malformed", args: []string{"--tensor-parallel-size"}, wantErr: "malformed"}, {name: "TP zero", args: []string{"--tensor-parallel-size=0"}, wantErr: "positive integer"}, {name: "PP two", args: []string{"--pipeline-parallel-size=2"}, wantErr: "pipeline parallel size 2"}, + {name: "PP malformed", args: []string{"--pipeline-parallel-size"}, wantErr: "pipeline parallelism"}, {name: "DP two", args: []string{"-dp", "2"}, wantErr: "data parallel size 2"}, + {name: "DP malformed", args: []string{"--data-parallel-size"}, wantErr: "data parallelism"}, {name: "external DP rank", args: []string{"--data-parallel-rank=0"}, wantErr: "multi-process data parallel flag"}, {name: "hybrid flag value", args: []string{"--disable-hybrid-kv-cache-manager=false"}, wantErr: "boolean flag"}, {name: "hybrid split value", args: []string{"--disable-hybrid-kv-cache-manager", "false"}, wantErr: "boolean flag"}, + {name: "duplicate hybrid flag", args: []string{"--disable-hybrid-kv-cache-manager", "--disable-hybrid-kv-cache-manager"}, wantErr: "duplicated"}, {name: "duplicate transfer config", args: []string{"--kv-transfer-config", "{}", "--kv-transfer-config={}"}, wantErr: "at most once"}, } for _, tc := range tests { @@ -174,11 +177,52 @@ func TestVLLMLMCacheMPValidateEngineParallelism(t *testing.T) { } } +func TestVLLMLMCacheMPValidateRejectsIncompleteTopology(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}).(runtimeadapter.LMCacheMPRuntimeAdapter) + validPod := newVLLMMPEnginePod() + tests := []struct { + name string + pod *corev1.Pod + cache *cachev1alpha1.CacheBackend + wantErr string + }{ + {name: "nil pod", cache: newTypedVLLMMPBackend(), wantErr: "pod is nil"}, + {name: "nil cache", pod: validPod, wantErr: "configuration is missing"}, + {name: "missing LMCache", pod: validPod, cache: &cachev1alpha1.CacheBackend{}, wantErr: "configuration is missing"}, + {name: "missing PodLocal server", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "PodLocal server configuration is missing"}, + {name: "missing NodeLocal server", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "NodeLocal server configuration is missing"}, + {name: "unsupported topology", pod: validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.Topology = "Remote" + return cache + }(), wantErr: "topology \"Remote\" is not implemented"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := adapter.ValidateMPEnginePod(tc.pod, tc.cache) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ValidateMPEnginePod error = %v, want substring %q", err, tc.wantErr) + } + }) + } +} + func TestVLLMLMCacheMPKVTransferConfigRoles(t *testing.T) { tests := []struct { role cachev1alpha1.CacheBackendIntegrationRole want string }{ + {role: cachev1alpha1.CacheBackendIntegrationRoleReadOnly, want: kvRoleConsumer}, + {role: cachev1alpha1.CacheBackendIntegrationRoleWriteOnly, want: kvRoleProducer}, {role: cachev1alpha1.CacheBackendIntegrationRoleReadWrite, want: kvRoleBoth}, {role: "", want: kvRoleBoth}, } @@ -206,6 +250,50 @@ func TestVLLMLMCacheMPKVTransferConfigRoles(t *testing.T) { } } +func TestVLLMLMCacheMPInjectRejectsInvalidInputs(t *testing.T) { + adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) + validPod := newVLLMMPEnginePod().Spec + tests := []struct { + name string + pod *corev1.PodSpec + binding *backendadapter.Binding + cache *cachev1alpha1.CacheBackend + wantErr string + }{ + {name: "nil pod", cache: newTypedVLLMMPBackend(), wantErr: "pod is nil"}, + {name: "nil cache", pod: &validPod, wantErr: "cache is nil"}, + {name: "missing LMCache", pod: &validPod, cache: &cachev1alpha1.CacheBackend{}, wantErr: "typed server configuration is required"}, + {name: "unsupported binding", pod: &validPod, cache: newTypedVLLMMPBackend(), binding: &backendadapter.Binding{Protocol: backendadapter.Protocol("grpc")}, wantErr: "does not support remote binding"}, + {name: "missing PodLocal server", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "typed PodLocal server configuration is required"}, + {name: "missing NodeLocal server", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.Topology = cachev1alpha1.LMCacheTopologyNodeLocal + cache.Spec.LMCache.PodLocal = nil + return cache + }(), wantErr: "typed NodeLocal server configuration is required"}, + {name: "unsupported topology", pod: &validPod, cache: func() *cachev1alpha1.CacheBackend { + cache := newTypedVLLMMPBackend() + cache.Spec.LMCache.Topology = "Remote" + return cache + }(), wantErr: "topology \"Remote\" is not implemented"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := adapter.InjectEngineConfig(tc.pod, tc.binding, tc.cache) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("InjectEngineConfig error = %v, want substring %q", err, tc.wantErr) + } + }) + } + if err := adapter.InjectRouterConfig(nil, nil, nil); err != nil { + t.Fatalf("InjectRouterConfig = %v, want nil", err) + } +} + func TestVLLMLMCacheMPInjectsCommonServerAndExternalConnector(t *testing.T) { adapter := NewVLLMLMCacheMPAdapter(SubscriberConfig{}) cache := newTypedVLLMMPBackend() diff --git a/internal/adapters/builtin/storage/effective_config_test.go b/internal/adapters/builtin/storage/effective_config_test.go new file mode 100644 index 00000000..f9c18f9a --- /dev/null +++ b/internal/adapters/builtin/storage/effective_config_test.go @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 The inference-cache Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package storage + +import ( + "testing" + + cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" +) + +func TestEffectiveProviderDefaultsHandleNilCache(t *testing.T) { + if got := effectiveProviderResources(nil); got != nil { + t.Fatalf("effectiveProviderResources(nil) = %+v, want nil", got) + } + if got := defaultServerResources(nil); len(got.Requests) != 0 || len(got.Limits) != 0 { + t.Fatalf("defaultServerResources(nil) = %+v, want empty requirements", got) + } + if got := effectiveProviderImage(nil, cachev1alpha1.CacheBackendRemoteStorageProviderRedis, "fallback:image"); got != "fallback:image" { + t.Fatalf("effectiveProviderImage(nil) = %q, want fallback:image", got) + } +} diff --git a/internal/controller/cachebackend_reconciler.go b/internal/controller/cachebackend_reconciler.go index b5efe5de..b05d8aca 100644 --- a/internal/controller/cachebackend_reconciler.go +++ b/internal/controller/cachebackend_reconciler.go @@ -30,19 +30,17 @@ import ( // DefaultMatchedEnginePodsRequeueInterval is the steady-state cadence at // which a CacheBackend with a configured spec.engineSelector self-requeues, // so the `status.matchedEnginePods` snapshot does not stay stale forever -// between otherwise-unrelated reconcile triggers. The reconciler does not -// Watch Pods by design (see refreshMatchedEnginePods godoc); without a -// self-requeue, the count would only refresh when the CR, the owned -// Deployment or Service changed. 30s strikes a balance between -// operator responsiveness and reconcile pressure on a large fleet. Tests -// override via the `MatchedEnginePodsRequeueInterval` reconciler field to -// avoid baking the 30s delay into the suite. +// if a Pod watch event is missed or coalesced. Pod events normally trigger an +// immediate reconcile; the 30s safety net balances eventual correction with +// reconcile pressure on a large fleet. Tests override via the +// `MatchedEnginePodsRequeueInterval` reconciler field to avoid baking the 30s +// delay into the suite. const DefaultMatchedEnginePodsRequeueInterval = 30 * time.Second // DefaultMatchedEnginePodsChurnRequeueInterval is the faster cadence used when // the observed pod count disagrees with the desired-replica sum of Deployments // whose pod-template labels match the CacheBackend's engineSelector. It keeps -// rolling restarts and scale churn visible without adding a Pod watch. +// rolling restarts and scale churn visible while Pod watch events converge. const DefaultMatchedEnginePodsChurnRequeueInterval = 5 * time.Second // CacheBackendReconciler reconciles a CacheBackend object. @@ -51,13 +49,11 @@ type CacheBackendReconciler struct { Scheme *runtime.Scheme Log logr.Logger Recorder events.EventRecorder - // APIReader is an uncached live client used for the per-reconcile pod - // List that backs status.matchedEnginePods. The cached client would - // register a Pod informer with controller-runtime, which the locked - // design explicitly rejected (would watch all pods cluster-wide - // just to count per-CR; the per-reconcile namespaced live List is - // cheaper at the cluster sizes we target). Production wiring passes - // mgr.GetAPIReader(); tests that don't exercise the + // APIReader is an uncached live client used for per-reconcile Pod lists that + // back engine demand and status. The Pod watch supplies prompt reconcile + // events, while the live reader avoids acting on an informer snapshot that + // has not yet observed the scheduling, readiness, or deletion event. + // Production wiring passes mgr.GetAPIReader(); tests that don't exercise the // matchedEnginePods writer can leave it nil (a nil APIReader makes // refreshMatchedEnginePods fall through to the embedded // client.Client so existing fake-client tests still work). diff --git a/pkg/adapters/backend/backend_test.go b/pkg/adapters/backend/backend_test.go index 032c1560..bd6ff4bd 100644 --- a/pkg/adapters/backend/backend_test.go +++ b/pkg/adapters/backend/backend_test.go @@ -5,12 +5,33 @@ package backend import ( + "strings" "testing" cachev1alpha1 "github.com/cachebox-project/inference-cache/api/v1alpha1" corev1 "k8s.io/api/core/v1" ) +func TestValidateExternalEndpointRejectsMissingProtocolAndAddress(t *testing.T) { + tests := []struct { + name string + provider cachev1alpha1.CacheBackendRemoteStorageProvider + endpoint string + want string + }{ + {name: "unsupported provider", provider: cachev1alpha1.CacheBackendRemoteStorageProvider("future"), endpoint: "cache.example:6379", want: "no endpoint protocol"}, + {name: "empty endpoint", provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, want: "endpoint is empty"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ValidateExternalEndpoint(tc.provider, tc.endpoint) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("ValidateExternalEndpoint error = %v, want substring %q", err, tc.want) + } + }) + } +} + func TestBindingForKeepsResolvedExternalEndpoint(t *testing.T) { storage := &cachev1alpha1.CacheBackendRemoteStorageSpec{ Provider: cachev1alpha1.CacheBackendRemoteStorageProviderRedis, From 18547b0e196d53efa8382fc6a48587e77a80bc6c Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Wed, 12 Aug 2026 23:33:08 -0700 Subject: [PATCH 12/13] Sync Pod watch documentation Signed-off-by: Yue Sun --- docs/design/cachebackend-api.md | 2 +- .../cachebackend_lmcache_mp_status.go | 7 ++++--- .../cachebackend_matched_pods_test.go | 6 +++--- internal/controller/cachebackend_status.go | 21 ++++++++----------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/docs/design/cachebackend-api.md b/docs/design/cachebackend-api.md index 9f0c0d49..892da039 100644 --- a/docs/design/cachebackend-api.md +++ b/docs/design/cachebackend-api.md @@ -694,7 +694,7 @@ Every newly admitted engine has exactly one CacheBackend owner. Every non-empty optional Redis tier is the fail-open boundary: with the default `spec.integration.failOpen: true`, Redis can degrade while the Pod-local L1 remains usable; `false` makes Redis a readiness dependency. -- The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); `EngineSelectorUnmatched` (Normal) when a configured selector first observes zero matching pods while engine pods are expected, transitions from matched to zero, or gains the diagnostic message during an upgrade from an older zero-count status. One advisory Event is recorded on the `CacheBackend` but triggered by engine-pod state rather than a CacheBackend condition transition: `InjectedEngineCrashLooping` (Warning) is emitted once when an injected engine pod's engine container is first observed in CrashLoopBackOff after connector injection — commonly a connector incompatibility (esp. a hybrid-attention model), surfaced as `EngineCompatibility=False/InjectedEngineCrashLooping`, but a crash-loop can also be a bad image/command/secret/OOM, so the cause is verified via the engine logs, not asserted by the Event. The controller does not watch engine pod status — it detects this on the next `CacheBackend` reconcile that lists the pods, so the Event reflects observation time, not the instant the container entered CrashLoopBackOff; a transient pod-list failure preserves the prior condition rather than re-firing it. +- The controller emits Events on the `CacheBackend` only on meaningful state changes, never on steady-state reconciles. Condition-transition-keyed Events: `BackendDegraded` (Warning) on entering `Conditions[Degraded]=True` with reason `ReplicasUnavailable` (the KV-event-gate `NoKVEventsObserved` flavor is suppressed — it carries its own event), `BackendRecovered` (Normal) on the transition back to `Ready=True` (similarly suppressed when recovering from `NoKVEventsObserved`, which carries its own `KVEventsObserved` event); the `FailClosedEnabled` / `FailOpenRestored` pair above; the KV-event readiness gate's `AwaitingFirstKVEvent` (Normal), `KVEventsObserved` (Normal), and `NoKVEventsObserved` (Warning); `EngineSelectorUnmatched` (Normal) when a configured selector first observes zero matching pods while engine pods are expected, transitions from matched to zero, or gains the diagnostic message during an upgrade from an older zero-count status. One advisory Event is recorded on the `CacheBackend` but triggered by engine-pod state rather than a CacheBackend condition transition: `InjectedEngineCrashLooping` (Warning) is emitted once when an injected engine pod's engine container is first observed in CrashLoopBackOff after connector injection — commonly a connector incompatibility (esp. a hybrid-attention model), surfaced as `EngineCompatibility=False/InjectedEngineCrashLooping`, but a crash-loop can also be a bad image/command/secret/OOM, so the cause is verified via the engine logs, not asserted by the Event. Engine and controller-owned NodeLocal-server Pod changes enqueue the owning `CacheBackend` immediately; each reconcile then lists Pods through the uncached API reader so lifecycle, coverage, readiness, and CrashLoop observations use an authoritative snapshot rather than a potentially lagging informer cache. Periodic self-requeues remain a bounded-staleness fallback for missed or coalesced watch events. The Event therefore reflects controller observation time, not necessarily the exact instant the container entered CrashLoopBackOff; a transient pod-list failure preserves the prior condition rather than re-firing it. - A `Normal InjectedByCacheBackend` Event is emitted on engine pods the mutating webhook stamps with both `inferencecache.io/injected-by` AND `inferencecache.io/injected-by-uid`, where the UID annotation matches the live CacheBackend's `metadata.uid` at reconcile time. The controller deliberately skips emission when (a) the named CR cannot be looked up (NotFound), (b) the UID annotation is absent (failurePolicy=Ignore forgery shape), or (c) the UID does not match the live CR (forgery or CR was recreated under the same name). Non-NotFound lookup errors surface as reconcile errors so controller-runtime retries with backoff. A pod explicitly opted out with a truthy `inferencecache.io/skip-inject` is instead stamped with `inferencecache.io/inject-skipped: skip-inject-annotation`; the same post-create controller emits a `Normal SkippedByOperator` Event only when both the truthy opt-out annotation and the webhook's skipped marker are present. The Events are recorded by a Pod-watching controller, not by the webhook itself: at mutating-admission time the apiserver hasn't assigned `metadata.uid` to the pod yet, so an event recorded from the webhook would carry `involvedObject.uid=""` and be invisible to describe (which filters events by UID). Routing the emission through a post-create controller is what guarantees the event reaches the user-visible surface. There is no `NoMatchingCacheBackend` Event; the no-match signals are `status.matchedEnginePods == 0`, `status.engineSelectorMessage`, and `EngineSelectorUnmatched` on the CacheBackend. - Optional nested specs are pointer fields in Go so omitted objects stay absent in JSON. `spec.integration` and `spec.observation` are the deliberate diff --git a/internal/controller/cachebackend_lmcache_mp_status.go b/internal/controller/cachebackend_lmcache_mp_status.go index 892780d1..8ce2cf12 100644 --- a/internal/controller/cachebackend_lmcache_mp_status.go +++ b/internal/controller/cachebackend_lmcache_mp_status.go @@ -55,9 +55,10 @@ func isTypedLMCachePodLocal(backend *cachev1alpha1.CacheBackend) bool { // from the optional remote L3. Native sidecars report their state under // status.initContainerStatuses, not containerStatuses. // -// Like matchedEnginePods, this is a bounded-cadence observation rather than a -// cluster-wide Pod watch. List/patch errors preserve the prior verdict and are -// fail-soft so connector observability cannot block normal reconciliation. +// Mapped engine and NodeLocal-server Pod changes normally trigger this +// observation immediately; periodic reconciliation is the bounded-staleness +// fallback. List/patch errors preserve the prior verdict and are fail-soft so +// connector observability cannot block normal reconciliation. func (r *CacheBackendReconciler) refreshLMCacheMPConnectorStatus(ctx context.Context, backend *cachev1alpha1.CacheBackend) { if !isTypedLMCacheMP(backend) { if backend.Status.Connector == nil && meta.FindStatusCondition(backend.Status.Conditions, conditionTypeConnectorReady) == nil { diff --git a/internal/controller/cachebackend_matched_pods_test.go b/internal/controller/cachebackend_matched_pods_test.go index bc92ede6..aec2c080 100644 --- a/internal/controller/cachebackend_matched_pods_test.go +++ b/internal/controller/cachebackend_matched_pods_test.go @@ -598,9 +598,9 @@ func TestReconcileMatchedEnginePodsCoexistsWithOtherStatusWriters(t *testing.T) // TestReconcileMatchedEnginePodsUsesAPIReaderForPods pins the structural // invariant the locked design called out: the pod List backing // matchedEnginePods MUST go through the manager's APIReader (uncached live -// client), not the manager's cached Client. Using Client would make -// controller-runtime register a cluster-wide Pod informer just to maintain -// this snapshot count — exactly what the "no Pod watch" rule rejects. +// client), not the manager's cached Client. The Pod watch supplies the trigger; +// the live List supplies an authoritative snapshot even when the informer that +// delivered that trigger has not yet converged. // // The test plumbs Client and APIReader to two DIFFERENT fake clients (the // Client carries the CB but ZERO pods; the APIReader carries ZERO CBs but diff --git a/internal/controller/cachebackend_status.go b/internal/controller/cachebackend_status.go index 5a55bbc1..14c24caf 100644 --- a/internal/controller/cachebackend_status.go +++ b/internal/controller/cachebackend_status.go @@ -698,18 +698,15 @@ type matchedEnginePodsRefresh struct { // reconciler and with any future status writers (e.g. an index-participation // projector) that touch different sub-fields. // -// Cadence-by-reconcile, not real-time: counts via a single namespaced -// client.List with the engineSelector — there is no Pod watch, and pod -// births/deaths between reconciles are not reflected until the next pass. -// To keep the count from going indefinitely stale between unrelated -// reconcile triggers, the Reconcile path sets `result.RequeueAfter = -// matchedEnginePodsRequeueInterval` whenever the CR has a non-empty -// EngineSelector, giving the field a bounded staleness without paying -// for a Pod informer. The real-time per-pod signal lives on the engine -// pods themselves (the `InjectedByCacheBackend` Event the -// engine-pod-events controller emits on every annotated pod); this -// status field answers the cluster-wide "is anyone connected at all?" -// question. +// Pod changes mapped to this CacheBackend normally trigger the refresh +// immediately. The count itself comes from one namespaced APIReader List with +// the engineSelector so reconciliation uses an authoritative snapshot rather +// than a potentially lagging informer cache. The periodic +// matchedEnginePodsRequeueInterval remains a bounded-staleness fallback for a +// missed or coalesced watch event. The real-time per-pod signal also lives on +// the engine pods themselves (the `InjectedByCacheBackend` Event the +// engine-pod-events controller emits on every annotated pod); this status field +// answers the cluster-wide "is anyone connected at all?" question. // // Selector resolution mirrors the mutating webhook's policy: a nil or // empty MatchLabels matches nothing (a broad selector at admission time From 464e911c195db5376e2e1e526dd38a55581ca459 Mon Sep 17 00:00:00 2001 From: Yue Sun Date: Thu, 13 Aug 2026 00:42:11 -0700 Subject: [PATCH 13/13] fix managed Redis auth entrypoint portability Signed-off-by: Yue Sun --- .../scripts/default_install_smoke.sh | 29 ++++++++++++++++++- internal/adapters/builtin/storage/redis.go | 12 ++++---- .../adapters/builtin/storage/redis_test.go | 15 +++++++--- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/docs/reference-stack/scripts/default_install_smoke.sh b/docs/reference-stack/scripts/default_install_smoke.sh index 8f9a5110..99ce72fb 100755 --- a/docs/reference-stack/scripts/default_install_smoke.sh +++ b/docs/reference-stack/scripts/default_install_smoke.sh @@ -171,6 +171,14 @@ kubectl create namespace "$SMOKE_NAMESPACE" --dry-run=client -o yaml | kubectl a log "creating typed PodLocal and NodeLocal MP backends" cat <<'EOF' | kubectl -n "$SMOKE_NAMESPACE" apply -f - >/dev/null +apiVersion: v1 +kind: Secret +metadata: + name: managed-redis-auth +type: Opaque +stringData: + password: install-smoke-password +--- apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend metadata: @@ -232,7 +240,11 @@ spec: nodeSelector: kubernetes.io/os: linux terminationGracePeriodSeconds: 45 - redis: {} + redis: + authentication: + password: + name: managed-redis-auth + key: password --- apiVersion: inferencecache.io/v1alpha1 kind: CacheBackend @@ -311,6 +323,21 @@ managed_os="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o json [ "$managed_os" = "linux" ] || fail "managed workload nodeSelector was not rendered" managed_grace="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}')" [ "$managed_grace" = "45" ] || fail "managed workload terminationGracePeriodSeconds was not rendered" +managed_command="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.containers[0].command}')" +[ -z "$managed_command" ] || fail "authenticated managed Redis overrides its image entrypoint: $managed_command" +managed_requirepass_arg="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.containers[0].args[-2]}')" +[ "$managed_requirepass_arg" = "--requirepass" ] || fail "authenticated managed Redis is missing --requirepass" +managed_password_arg="$(kubectl -n "$SMOKE_NAMESPACE" get deployment managed-redis -o jsonpath='{.spec.template.spec.containers[0].args[-1]}')" +[ "$managed_password_arg" = '$(REDIS_PASSWORD)' ] || fail "authenticated managed Redis does not use the Secret-backed password environment reference" +kubectl -n "$SMOKE_NAMESPACE" rollout status deployment/managed-redis --timeout="$READY_TIMEOUT" >/dev/null \ + || fail "authenticated managed Redis did not become available" +managed_pod="$(kubectl -n "$SMOKE_NAMESPACE" get pod -l app.kubernetes.io/instance=managed-redis -o jsonpath='{.items[0].metadata.name}')" +[ -n "$managed_pod" ] || fail "authenticated managed Redis Pod was not found" +unauthenticated_ping="$(kubectl -n "$SMOKE_NAMESPACE" exec "$managed_pod" -- sh -c 'unset REDISCLI_AUTH; redis-cli ping' 2>&1 || true)" +grep -Fq 'NOAUTH Authentication required' <<<"$unauthenticated_ping" \ + || fail "managed Redis accepted an unauthenticated PING: $unauthenticated_ping" +[ "$(kubectl -n "$SMOKE_NAMESPACE" exec "$managed_pod" -- redis-cli ping)" = "PONG" ] \ + || fail "managed Redis Secret-backed authenticated PING failed" log "checking NodeLocal creates no speculative server before an engine is scheduled" [ "$(kubectl -n "$SMOKE_NAMESPACE" get pods -l inferencecache.io/lmcache-node-server=true --no-headers 2>/dev/null | wc -l | tr -d ' ')" = "0" ] \ diff --git a/internal/adapters/builtin/storage/redis.go b/internal/adapters/builtin/storage/redis.go index c429fc5e..8ed0636f 100644 --- a/internal/adapters/builtin/storage/redis.go +++ b/internal/adapters/builtin/storage/redis.go @@ -129,9 +129,10 @@ func ResolveRedisL2Server(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, * // Managed authentication configures BOTH ends of the binding: this Redis // process requires the Secret-backed password, while the MP renderer maps // the same selector to LMCACHE_RESP_PASSWORD. The secret value never enters - // the PodSpec or CacheBackend status. The shell expands it only inside the - // container and then explicitly invokes the official image entrypoint, which - // preserves its root-to-redis privilege drop. + // the PodSpec or CacheBackend status. Kubelet expands the environment + // reference in Args before invoking the image's own entrypoint, so managed + // authentication does not depend on the official Redis image's private + // docker-entrypoint.sh path. if storage := cache.Spec.EffectiveRemoteStorage(); storage != nil && storage.Redis != nil && storage.Redis.Authentication != nil { redis := storage.Redis auth := redis.Authentication @@ -143,10 +144,7 @@ func ResolveRedisL2Server(cache *cachev1alpha1.CacheBackend) (*corev1.PodSpec, * corev1.EnvVar{Name: redisPasswordEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: passwordRef}}, corev1.EnvVar{Name: redisCLIAuthEnv, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: auth.Password.DeepCopy()}}, ) - const authScript = `set -eu -exec /usr/local/bin/docker-entrypoint.sh "$@" --requirepass "$REDIS_PASSWORD"` - container.Command = []string{"/bin/sh", "-c"} - container.Args = append([]string{authScript, "inference-cache-redis-auth"}, container.Args...) + container.Args = append(container.Args, "--requirepass", "$("+redisPasswordEnv+")") // REDISCLI_AUTH lets redis-cli authenticate without placing the password // in the probe command. A TCP-only probe would declare a server Ready even // if AUTH setup were unusable. diff --git a/internal/adapters/builtin/storage/redis_test.go b/internal/adapters/builtin/storage/redis_test.go index 7a82c747..1018ece7 100644 --- a/internal/adapters/builtin/storage/redis_test.go +++ b/internal/adapters/builtin/storage/redis_test.go @@ -188,6 +188,7 @@ func TestResolveRedisL2ServerImageOverride(t *testing.T) { func TestResolveRedisL2ServerPasswordAuthentication(t *testing.T) { cb := newCacheBackend(cachev1alpha1.CacheBackendTypeLMCache, "sglang") + cb.Spec.RemoteStorage.Redis.Image = "registry.example/redis-compatible@sha256:deadbeef" cb.Spec.RemoteStorage.Redis.Authentication = &cachev1alpha1.RedisAuthenticationSpec{ Password: corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: "credential-source"}, @@ -199,12 +200,18 @@ func TestResolveRedisL2ServerPasswordAuthentication(t *testing.T) { t.Fatalf("ResolveRedisL2Server: %v", err) } c := pod.Containers[0] - if len(c.Command) != 2 || c.Command[0] != "/bin/sh" || c.Command[1] != "-c" { - t.Fatalf("authenticated Redis command = %v", c.Command) + if c.Image != "registry.example/redis-compatible@sha256:deadbeef" { + t.Fatalf("authenticated Redis image = %q, want custom compatible image", c.Image) + } + if len(c.Command) != 0 { + t.Fatalf("authenticated Redis command = %v, want the custom image entrypoint preserved", c.Command) + } + if password, found := argVal(c.Args, "--requirepass"); !found || password != "$(REDIS_PASSWORD)" { + t.Fatalf("authenticated Redis --requirepass = %q (found=%v), want kubelet-expanded environment reference", password, found) } joined := strings.Join(c.Args, " ") - if !strings.Contains(joined, "/usr/local/bin/docker-entrypoint.sh") || !strings.Contains(joined, `--requirepass "$REDIS_PASSWORD"`) { - t.Fatalf("authenticated Redis args = %s", joined) + if strings.Contains(joined, "docker-entrypoint.sh") || strings.Contains(joined, "/bin/sh") { + t.Fatalf("authenticated Redis args depend on an image-private shell or entrypoint: %s", joined) } if strings.Contains(joined, "credential-source") { t.Fatalf("secret name leaked into args: %s", joined)