From 254fba2b66fce7322084c3b73eae8a6b3250e59a Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 10:28:36 -0400 Subject: [PATCH 1/7] feat(gateway-provisioning): Adding deployment readiness timeout env var, with default of 10 minutes Co-Authored-By: Claude Opus 4.6 (1M context) --- .../control-plane/internal/reconciler/health.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index e35b0e277..7124de320 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -30,6 +30,8 @@ const defaultHealthInterval = 30 * time.Second // specs/platform/openshell-gateway-routing.spec.md ยง Gateway Exposure Configuration. const defaultRouteReadyTimeout = 10 * time.Minute +const defaultDeploymentReadyTimeout = 10 * time.Minute + // routeVerifyInterval is the minimum time between residual route/console // absence re-checks for a settled (torn-down, addressless) gateway. // @@ -150,6 +152,16 @@ func routeReadyTimeout() time.Duration { return defaultRouteReadyTimeout } +func deploymentReadyTimeout() time.Duration { + if v := os.Getenv("GATEWAY_DEPLOYMENT_READY_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + return d + } + log.Printf("WARN invalid GATEWAY_DEPLOYMENT_READY_TIMEOUT %q; using default %s", v, defaultDeploymentReadyTimeout) + } + return defaultDeploymentReadyTimeout +} + // Run drives the health reconciliation loop until the context is cancelled. func (h *GatewayHealthReconciler) Run(ctx context.Context) error { log.Printf("INFO gateway health reconciler started (interval=%s routeReadyTimeout=%s ingressMode=%s)", h.interval, h.routeReadyTimeout, h.ingressMode) From 19a830c6b25e36a4ef3d164357245008fff07da9 Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 10:30:37 -0400 Subject: [PATCH 2/7] feat(gateway-provisioning): Appending deployment readiness metrics to the deployment struct Co-Authored-By: Claude Opus 4.6 (1M context) --- .../internal/reconciler/health.go | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index 7124de320..e9addaedc 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -57,7 +57,8 @@ type GatewayHealthReconciler struct { clusterID string interval time.Duration exposure exposure.Port - routeReadyTimeout time.Duration + routeReadyTimeout time.Duration + deploymentReadyTimeout time.Duration keycloakConfig *gateway.KeycloakConfig isOpenShift bool hasGatewayAPI bool @@ -96,10 +97,11 @@ type GatewayHealthReconciler struct { // settled gateway keeps being re-verified forever at that low cadence, because // elapsed wall-clock time is not proof that a stale provisioning pass cannot // still resurrect resources (see routeVerifyInterval). - mu sync.Mutex - routeNotReadySince map[string]time.Time - routeTornDown map[string]bool - routeVerifiedAt map[string]time.Time + mu sync.Mutex + routeNotReadySince map[string]time.Time + deploymentNotReadySince map[string]time.Time + routeTornDown map[string]bool + routeVerifiedAt map[string]time.Time } func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient dynamic.Interface, grpcConn *grpc.ClientConn, exposurePort exposure.Port, keycloakConfig *gateway.KeycloakConfig, clusterID string) *GatewayHealthReconciler { @@ -126,17 +128,19 @@ func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient d clusterID: clusterID, interval: defaultHealthInterval, exposure: exposurePort, - routeReadyTimeout: routeReadyTimeout(), - keycloakConfig: keycloakConfig, + routeReadyTimeout: routeReadyTimeout(), + deploymentReadyTimeout: deploymentReadyTimeout(), + keycloakConfig: keycloakConfig, consoleClientChecker: consoleClientChecker, isOpenShift: isOpenShift, hasGatewayAPI: hasGatewayAPI, ingressMode: ingressMode, skipNetworkPolicies: os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true", now: time.Now, - routeNotReadySince: make(map[string]time.Time), - routeTornDown: make(map[string]bool), - routeVerifiedAt: make(map[string]time.Time), + routeNotReadySince: make(map[string]time.Time), + deploymentNotReadySince: make(map[string]time.Time), + routeTornDown: make(map[string]bool), + routeVerifiedAt: make(map[string]time.Time), } } From cce291496f07bb220b526b403927dd4890dd9b85 Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 10:38:00 -0400 Subject: [PATCH 3/7] feat(gateway-provisioning): Adding reconciler helpers, marking deployments as ready or unready, clearing deployment timer for gateway Co-Authored-By: Claude Opus 4.6 (1M context) --- .../control-plane/internal/reconciler/health.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index e9addaedc..58d2e7136 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -591,3 +591,20 @@ func (h *GatewayHealthReconciler) clearRouteTimer(gatewayID string) { defer h.mu.Unlock() delete(h.routeNotReadySince, gatewayID) } + +func (h *GatewayHealthReconciler) markDeploymentNotReady(gatewayID string) time.Time { + h.mu.Lock() + defer h.mu.Unlock() + if t, ok := h.deploymentNotReadySince[gatewayID]; ok { + return t + } + t := h.now() + h.deploymentNotReadySince[gatewayID] = t + return t +} + +func (h *GatewayHealthReconciler) clearDeploymentTimer(gatewayID string) { + h.mu.Lock() + defer h.mu.Unlock() + delete(h.deploymentNotReadySince, gatewayID) +} From 98f068ec44464b86b78ff62bc8ff6d14665589f8 Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 10:42:57 -0400 Subject: [PATCH 4/7] feat(gateway-readiness): Refactoring gateway not-ready logic, using time deference and statefulness to track gateway teardown Co-Authored-By: Claude Opus 4.6 (1M context) --- .../internal/reconciler/health.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index 58d2e7136..41bcf5b3b 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -258,23 +258,30 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl var desiredPhase, desiredStatus string switch { case !ready: - // The Deployment has not been created yet; the provisioning path still - // owns this gateway. Leave its phase untouched. if reason == "deployment not found" { return } h.clearRouteTimer(gatewayID) - desiredPhase, desiredStatus = string(gatewayhealth.PhaseDegraded), reason + if phase == string(gatewayhealth.PhaseProvisioning) { + since := h.markDeploymentNotReady(gatewayID) + if h.now().Sub(since) >= h.deploymentReadyTimeout { + h.clearDeploymentTimer(gatewayID) + desiredPhase, desiredStatus = string(gatewayhealth.PhaseDegraded), fmt.Sprintf("deployment not ready after %s: %s", h.deploymentReadyTimeout, reason) + } else { + desiredPhase, desiredStatus = string(gatewayhealth.PhaseProvisioning), reason + } + } else { + h.clearDeploymentTimer(gatewayID) + desiredPhase, desiredStatus = string(gatewayhealth.PhaseDegraded), reason + } case h.exposure != nil && isRoutedGateway(gw): - // Deployment is Ready; a routed gateway additionally requires its external - // exposure to be observed Ready before it can be Running. + h.clearDeploymentTimer(gatewayID) desiredPhase, desiredStatus = h.evaluateRouteReadiness(ctx, gatewayID, namespace, phase) if desiredPhase == "" { - // Transient error observing the exposure; leave the phase untouched - // rather than flap the gateway. return } default: + h.clearDeploymentTimer(gatewayID) h.clearRouteTimer(gatewayID) desiredPhase, desiredStatus = string(gatewayhealth.PhaseRunning), gatewayhealth.StatusHealthy } From 4536a798152b6c596aa40f87d641e6dbb4a02c6e Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 12:52:13 -0400 Subject: [PATCH 5/7] feat(gateway-provisioning): Adding gateway readiness function to signature, updating health check unit tests Co-Authored-By: Claude Opus 4.6 (1M context) --- .../internal/reconciler/health.go | 9 +- .../internal/reconciler/health_test.go | 103 +++++++++++++++++- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index 41bcf5b3b..2fbfb3b80 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -74,8 +74,8 @@ type GatewayHealthReconciler struct { // client needs no additional synchronization. consoleClientChecker gateway.ConsoleClientChecker - // now is the clock, overridable in tests. - now func() time.Time + now func() time.Time + deploymentReadinessFn func(ctx context.Context, clientset kubernetes.Interface, namespace, name string) (bool, string, error) // routeNotReadySince records, per gateway, when its Deployment first became // Ready while its external exposure was not, so the route-readiness grace @@ -136,7 +136,8 @@ func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient d hasGatewayAPI: hasGatewayAPI, ingressMode: ingressMode, skipNetworkPolicies: os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true", - now: time.Now, + now: time.Now, + deploymentReadinessFn: gateway.DeploymentReadiness, routeNotReadySince: make(map[string]time.Time), deploymentNotReadySince: make(map[string]time.Time), routeTornDown: make(map[string]bool), @@ -249,7 +250,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl log.Printf("WARN gateway health: %s: %v", gatewayID, err) return } - ready, reason, err := gateway.DeploymentReadiness(ctx, h.clientset, namespace, gateway.GatewayDeploymentName) + ready, reason, err := h.deploymentReadinessFn(ctx, h.clientset, namespace, gateway.GatewayDeploymentName) if err != nil { log.Printf("WARN gateway health: %s: %v", gatewayID, err) return diff --git a/components/control-plane/internal/reconciler/health_test.go b/components/control-plane/internal/reconciler/health_test.go index de56b15a7..d33d84a0f 100644 --- a/components/control-plane/internal/reconciler/health_test.go +++ b/components/control-plane/internal/reconciler/health_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes" k8sfake "k8s.io/client-go/kubernetes/fake" ) @@ -36,10 +37,12 @@ func (f fakeExposure) ObserveReadiness(context.Context, exposure.Request) (expos func newHealthRec(exp exposure.Port, now func() time.Time, timeout time.Duration) *GatewayHealthReconciler { return &GatewayHealthReconciler{ - exposure: exp, - routeReadyTimeout: timeout, - now: now, - routeNotReadySince: make(map[string]time.Time), + exposure: exp, + routeReadyTimeout: timeout, + deploymentReadyTimeout: timeout, + now: now, + routeNotReadySince: make(map[string]time.Time), + deploymentNotReadySince: make(map[string]time.Time), } } @@ -676,3 +679,95 @@ func TestListAllGateways_Empty(t *testing.T) { t.Fatalf("expected 1 call, got %d", callCount) } } + +func fakeDeploymentReadiness(ready bool, reason string) func(context.Context, kubernetes.Interface, string, string) (bool, string, error) { + return func(context.Context, kubernetes.Interface, string, string) (bool, string, error) { + return ready, reason, nil + } +} + +func newHealthRecForDeploymentTest(now func() time.Time, timeout time.Duration, readyFn func(context.Context, kubernetes.Interface, string, string) (bool, string, error)) *GatewayHealthReconciler { + h := newHealthRec(nil, now, timeout) + h.deploymentReadinessFn = readyFn + h.routeTornDown = map[string]bool{"gw-1": true} + h.routeVerifiedAt = map[string]time.Time{"gw-1": now()} + return h +} + +func TestReconcileGatewayHealth_ProvisioningDeploymentNotReadyWithinGraceStaysProvisioning(t *testing.T) { + var gotPhase string + client := &fakeGatewayClient{updateFn: func(_ context.Context, req *pb.UpdateGatewayRequest, _ ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + if req.Phase != nil { + gotPhase = *req.Phase + } + return &pb.UpdateGatewayResponse{}, nil + }} + + phase := "Provisioning" + gw := &pb.Gateway{ + Metadata: &pb.ObjectReference{Id: "gw-1"}, + Phase: &phase, + Namespace: "openshell-abc", + } + + h := newHealthRecForDeploymentTest(fixedClock(time.Unix(1000, 0)), 10*time.Minute, fakeDeploymentReadiness(false, "0/1 replicas ready")) + h.reconcileGatewayHealth(context.Background(), client, gw) + + if gotPhase != "Provisioning" { + t.Fatalf("got phase %q, want Provisioning (within grace window)", gotPhase) + } +} + +func TestReconcileGatewayHealth_ProvisioningDeploymentNotReadyBeyondGraceBecomesDegraded(t *testing.T) { + var gotPhase string + client := &fakeGatewayClient{updateFn: func(_ context.Context, req *pb.UpdateGatewayRequest, _ ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + if req.Phase != nil { + gotPhase = *req.Phase + } + return &pb.UpdateGatewayResponse{}, nil + }} + + phase := "Provisioning" + gw := &pb.Gateway{ + Metadata: &pb.ObjectReference{Id: "gw-1"}, + Phase: &phase, + Namespace: "openshell-abc", + } + + cur := time.Unix(1000, 0) + h := newHealthRecForDeploymentTest(func() time.Time { return cur }, 10*time.Minute, fakeDeploymentReadiness(false, "0/1 replicas ready")) + + h.reconcileGatewayHealth(context.Background(), client, gw) + + cur = cur.Add(11 * time.Minute) + h.routeVerifiedAt["gw-1"] = cur + h.reconcileGatewayHealth(context.Background(), client, gw) + + if gotPhase != "Degraded" { + t.Fatalf("got phase %q, want Degraded after grace window expired", gotPhase) + } +} + +func TestReconcileGatewayHealth_RunningLosesDeploymentReadinessBecomesDegradedImmediately(t *testing.T) { + var gotPhase string + client := &fakeGatewayClient{updateFn: func(_ context.Context, req *pb.UpdateGatewayRequest, _ ...grpc.CallOption) (*pb.UpdateGatewayResponse, error) { + if req.Phase != nil { + gotPhase = *req.Phase + } + return &pb.UpdateGatewayResponse{}, nil + }} + + phase := "Running" + gw := &pb.Gateway{ + Metadata: &pb.ObjectReference{Id: "gw-1"}, + Phase: &phase, + Namespace: "openshell-abc", + } + + h := newHealthRecForDeploymentTest(fixedClock(time.Unix(1000, 0)), 10*time.Minute, fakeDeploymentReadiness(false, "0/1 replicas ready")) + h.reconcileGatewayHealth(context.Background(), client, gw) + + if gotPhase != "Degraded" { + t.Fatalf("got phase %q, want Degraded immediately (no grace for Running)", gotPhase) + } +} From f90bbbf8c2c935c3f85cb19999861a7512d55729 Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 14:11:38 -0400 Subject: [PATCH 6/7] fix(gateway-provisioning): Go lint corrections --- .../internal/reconciler/health.go | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/components/control-plane/internal/reconciler/health.go b/components/control-plane/internal/reconciler/health.go index 2fbfb3b80..6928cae84 100644 --- a/components/control-plane/internal/reconciler/health.go +++ b/components/control-plane/internal/reconciler/health.go @@ -54,16 +54,16 @@ type GatewayHealthReconciler struct { // clusterID scopes the health sweep to this managed cluster's gateways. When // non-empty the fleet list is filtered server-side so a spoke never stamps // (Degraded/Running) a gateway owned by another cluster. Empty sweeps all. - clusterID string - interval time.Duration - exposure exposure.Port + clusterID string + interval time.Duration + exposure exposure.Port routeReadyTimeout time.Duration deploymentReadyTimeout time.Duration - keycloakConfig *gateway.KeycloakConfig - isOpenShift bool - hasGatewayAPI bool - ingressMode string - skipNetworkPolicies bool + keycloakConfig *gateway.KeycloakConfig + isOpenShift bool + hasGatewayAPI bool + ingressMode string + skipNetworkPolicies bool // consoleClientChecker is a single, long-lived Keycloak client reused across // every tick's residual-absence checks. Constructed once (when Keycloak is @@ -97,11 +97,11 @@ type GatewayHealthReconciler struct { // settled gateway keeps being re-verified forever at that low cadence, because // elapsed wall-clock time is not proof that a stale provisioning pass cannot // still resurrect resources (see routeVerifyInterval). - mu sync.Mutex - routeNotReadySince map[string]time.Time - deploymentNotReadySince map[string]time.Time - routeTornDown map[string]bool - routeVerifiedAt map[string]time.Time + mu sync.Mutex + routeNotReadySince map[string]time.Time + deploymentNotReadySince map[string]time.Time + routeTornDown map[string]bool + routeVerifiedAt map[string]time.Time } func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient dynamic.Interface, grpcConn *grpc.ClientConn, exposurePort exposure.Port, keycloakConfig *gateway.KeycloakConfig, clusterID string) *GatewayHealthReconciler { @@ -122,22 +122,22 @@ func NewGatewayHealthReconciler(clientset *kubernetes.Clientset, dynamicClient d hasGatewayAPI := gateway.DetectGatewayAPI(clientset) ingressMode := gateway.IngressMode(hasGatewayAPI, isOpenShift) return &GatewayHealthReconciler{ - clientset: clientset, - dynamicClient: dynamicClient, - grpcConn: grpcConn, - clusterID: clusterID, - interval: defaultHealthInterval, - exposure: exposurePort, - routeReadyTimeout: routeReadyTimeout(), - deploymentReadyTimeout: deploymentReadyTimeout(), - keycloakConfig: keycloakConfig, - consoleClientChecker: consoleClientChecker, - isOpenShift: isOpenShift, - hasGatewayAPI: hasGatewayAPI, - ingressMode: ingressMode, - skipNetworkPolicies: os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true", - now: time.Now, - deploymentReadinessFn: gateway.DeploymentReadiness, + clientset: clientset, + dynamicClient: dynamicClient, + grpcConn: grpcConn, + clusterID: clusterID, + interval: defaultHealthInterval, + exposure: exposurePort, + routeReadyTimeout: routeReadyTimeout(), + deploymentReadyTimeout: deploymentReadyTimeout(), + keycloakConfig: keycloakConfig, + consoleClientChecker: consoleClientChecker, + isOpenShift: isOpenShift, + hasGatewayAPI: hasGatewayAPI, + ingressMode: ingressMode, + skipNetworkPolicies: os.Getenv("GATEWAY_SKIP_NETWORK_POLICIES") == "true", + now: time.Now, + deploymentReadinessFn: gateway.DeploymentReadiness, routeNotReadySince: make(map[string]time.Time), deploymentNotReadySince: make(map[string]time.Time), routeTornDown: make(map[string]bool), From 8fd21e51af4f412896f1b0980d763703e7c1d7e9 Mon Sep 17 00:00:00 2001 From: wesgreen Date: Wed, 9 Sep 2026 14:52:05 -0400 Subject: [PATCH 7/7] feat(gateway-provisioning): Updating spec with timeout env var --- specs/platform/openshell-gateway-health.spec.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/platform/openshell-gateway-health.spec.md b/specs/platform/openshell-gateway-health.spec.md index ff05c9b46..d8625892d 100644 --- a/specs/platform/openshell-gateway-health.spec.md +++ b/specs/platform/openshell-gateway-health.spec.md @@ -121,6 +121,17 @@ non-empty `.status.addresses`. - AND it SHALL set the `phase` to `Degraded` - AND it SHALL record the reason in `status` +### Requirement: Deployment Readiness Grace Configuration + +| Variable | Default | Description | +|---|---|---| +| `GATEWAY_DEPLOYMENT_READY_TIMEOUT` | `10m` | Grace window a Provisioning gateway's Deployment may remain not-Ready before the control plane transitions the `phase` to `Degraded`. A gateway that had already reached `Running` and then loses Deployment readiness is moved to `Degraded` immediately, with no grace. | + +The grace window governs only the `Provisioning -> Degraded` transition for a +Deployment that never becomes Ready; it is not a hard deadline that stops +observation. The control plane SHALL keep observing the Deployment after the +window elapses, so a gateway that eventually becomes ready returns to `Running`. + #### Scenario: Provisioning fails to apply - GIVEN a Gateway being reconciled