MIR-858: Surface sandbox failures in app status and doctor

api/app/app_v1alpha/rpc.gen.go

@@ -1235,6 +1235,170 @@ func (v *PoolStatus) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &v.data)
 }
 
+type serviceHealthData struct {
+	Service            *string             `cbor:"0,keyasint,omitempty" json:"service,omitempty"`
+	Health             *string             `cbor:"1,keyasint,omitempty" json:"health,omitempty"`
+	Running            *int32              `cbor:"2,keyasint,omitempty" json:"running,omitempty"`
+	Dead               *int32              `cbor:"3,keyasint,omitempty" json:"dead,omitempty"`
+	CrashCount         *int64              `cbor:"4,keyasint,omitempty" json:"crash_count,omitempty"`
+	CooldownSeconds    *int32              `cbor:"5,keyasint,omitempty" json:"cooldown_seconds,omitempty"`
+	LastExitCode       *int64              `cbor:"6,keyasint,omitempty" json:"last_exit_code,omitempty"`
+	LastFailureSandbox *string             `cbor:"7,keyasint,omitempty" json:"last_failure_sandbox,omitempty"`
+	LastFailureAt      *standard.Timestamp `cbor:"8,keyasint,omitempty" json:"last_failure_at,omitempty"`
+}
+
+type ServiceHealth struct {
+	data serviceHealthData
+}
+
+func (v *ServiceHealth) HasService() bool {
+	return v.data.Service != nil
+}
+
+func (v *ServiceHealth) Service() string {
+	if v.data.Service == nil {
+		return ""
+	}
+	return *v.data.Service
+}
+
+func (v *ServiceHealth) SetService(service string) {
+	v.data.Service = &service
+}
+
+func (v *ServiceHealth) HasHealth() bool {
+	return v.data.Health != nil
+}
+
+func (v *ServiceHealth) Health() string {
+	if v.data.Health == nil {
+		return ""
+	}
+	return *v.data.Health
+}
+
+func (v *ServiceHealth) SetHealth(health string) {
+	v.data.Health = &health
+}
+
+func (v *ServiceHealth) HasRunning() bool {
+	return v.data.Running != nil
+}
+
+func (v *ServiceHealth) Running() int32 {
+	if v.data.Running == nil {
+		return 0
+	}
+	return *v.data.Running
+}
+
+func (v *ServiceHealth) SetRunning(running int32) {
+	v.data.Running = &running
+}
+
+func (v *ServiceHealth) HasDead() bool {
+	return v.data.Dead != nil
+}
+
+func (v *ServiceHealth) Dead() int32 {
+	if v.data.Dead == nil {
+		return 0
+	}
+	return *v.data.Dead
+}
+
+func (v *ServiceHealth) SetDead(dead int32) {
+	v.data.Dead = &dead
+}
+
+func (v *ServiceHealth) HasCrashCount() bool {
+	return v.data.CrashCount != nil
+}
+
+func (v *ServiceHealth) CrashCount() int64 {
+	if v.data.CrashCount == nil {
+		return 0
+	}
+	return *v.data.CrashCount
+}
+
+func (v *ServiceHealth) SetCrashCount(crashCount int64) {
+	v.data.CrashCount = &crashCount
+}
+
+func (v *ServiceHealth) HasCooldownSeconds() bool {
+	return v.data.CooldownSeconds != nil
+}
+
+func (v *ServiceHealth) CooldownSeconds() int32 {
+	if v.data.CooldownSeconds == nil {
+		return 0
+	}
+	return *v.data.CooldownSeconds
+}
+
+func (v *ServiceHealth) SetCooldownSeconds(cooldownSeconds int32) {
+	v.data.CooldownSeconds = &cooldownSeconds
+}
+
+func (v *ServiceHealth) HasLastExitCode() bool {
+	return v.data.LastExitCode != nil
+}
+
+func (v *ServiceHealth) LastExitCode() int64 {
+	if v.data.LastExitCode == nil {
+		return 0
+	}
+	return *v.data.LastExitCode
+}
+
+func (v *ServiceHealth) SetLastExitCode(lastExitCode int64) {
+	v.data.LastExitCode = &lastExitCode
+}
+
+func (v *ServiceHealth) HasLastFailureSandbox() bool {
+	return v.data.LastFailureSandbox != nil
+}
+
+func (v *ServiceHealth) LastFailureSandbox() string {
+	if v.data.LastFailureSandbox == nil {
+		return ""
+	}
+	return *v.data.LastFailureSandbox
+}
+
+func (v *ServiceHealth) SetLastFailureSandbox(lastFailureSandbox string) {
+	v.data.LastFailureSandbox = &lastFailureSandbox
+}
+
+func (v *ServiceHealth) HasLastFailureAt() bool {
+	return v.data.LastFailureAt != nil
+}
+
+func (v *ServiceHealth) LastFailureAt() *standard.Timestamp {
+	return v.data.LastFailureAt
+}
+
+func (v *ServiceHealth) SetLastFailureAt(lastFailureAt *standard.Timestamp) {
+	v.data.LastFailureAt = lastFailureAt
+}
+
+func (v *ServiceHealth) MarshalCBOR() ([]byte, error) {
+	return cbor.Marshal(v.data)
+}
+
+func (v *ServiceHealth) UnmarshalCBOR(data []byte) error {
+	return cbor.Unmarshal(data, &v.data)
+}
+
+func (v *ServiceHealth) MarshalJSON() ([]byte, error) {
+	return json.Marshal(v.data)
+}
+
+func (v *ServiceHealth) UnmarshalJSON(data []byte) error {
+	return json.Unmarshal(data, &v.data)
+}
+
 type windowStatusData struct {
 	Version *string `cbor:"0,keyasint,omitempty" json:"version,omitempty"`
 	Leases  *int32  `cbor:"1,keyasint,omitempty" json:"leases,omitempty"`
@@ -1329,6 +1493,7 @@ type applicationStatusData struct {
 	BoundPorts        *[]*BoundPort       `cbor:"19,keyasint,omitempty" json:"bound_ports,omitempty"`
 	WorkloadRole      *string             `cbor:"20,keyasint,omitempty" json:"workload_role,omitempty"`
 	MaintenanceRoutes *[]string           `cbor:"21,keyasint,omitempty" json:"maintenance_routes,omitempty"`
+	Services          *[]*ServiceHealth   `cbor:"22,keyasint,omitempty" json:"services,omitempty"`
 }
 
 type ApplicationStatus struct {
@@ -1671,6 +1836,22 @@ func (v *ApplicationStatus) SetMaintenanceRoutes(maintenanceRoutes []string) {
 	v.data.MaintenanceRoutes = &x
 }
 
+func (v *ApplicationStatus) HasServices() bool {
+	return v.data.Services != nil
+}
+
+func (v *ApplicationStatus) Services() []*ServiceHealth {
+	if v.data.Services == nil {
+		return nil
+	}
+	return *v.data.Services
+}
+
+func (v *ApplicationStatus) SetServices(services []*ServiceHealth) {
+	x := slices.Clone(services)
+	v.data.Services = &x
+}
+
 func (v *ApplicationStatus) MarshalCBOR() ([]byte, error) {
 	return cbor.Marshal(v.data)
 }

api/app/rpc.yml

@@ -258,6 +258,37 @@ types:
         type: int64
         index: 3
 
+  - type: ServiceHealth
+    fields:
+      - name: service
+        type: string
+        index: 0
+      - name: health
+        type: string
+        index: 1
+      - name: running
+        type: int32
+        index: 2
+      - name: dead
+        type: int32
+        index: 3
+      - name: crashCount
+        type: int64
+        index: 4
+      - name: cooldownSeconds
+        type: int32
+        index: 5
+      - name: lastExitCode
+        type: int64
+        index: 6
+        optional: true
+      - name: lastFailureSandbox
+        type: string
+        index: 7
+      - name: lastFailureAt
+        type: standard.Timestamp
+        index: 8
+
   - type: WindowStatus
     fields:
       - name: version
@@ -359,6 +390,10 @@ types:
         element: string
         index: 21
         doc: "Hostnames of this app's routes currently in maintenance, serving a holding page instead of the app. The default route appears as an empty string."
+      - name: services
+        type: list
+        element: '*ServiceHealth'
+        index: 22
 
   - type: LogEntry
     fields:

cli/commands/app_status.go

@@ -62,10 +62,11 @@ func AppStatus(ctx *Context, opts struct {
 	if err != nil {
 		recentResult = nil
 	}
+	health, healthErr := fetchServiceHealth(ctx, opts.App)
 
 	// JSON output
 	if opts.IsJSON() {
-		return printAppStatusJSON(appResult, appConfig, activeDeployment, recentResult, opts.App, clusterId)
+		return printAppStatusJSON(appResult, appConfig, activeDeployment, recentResult, opts.App, clusterId, health, healthErr)
 	}
 
 	// Define styles
@@ -231,6 +232,16 @@ func AppStatus(ctx *Context, opts struct {
 		ctx.Printf("\n%s\n", yellowStyle.Render("No active deployment found"))
 	}
 
+	ctx.Printf("\n%s\n", labelStyle.Render("Service Sandboxes:"))
+	switch {
+	case healthErr != nil:
+		ctx.Printf("  Health unavailable: %v\n", healthErr)
+	case len(health) == 0:
+		ctx.Printf("  No service pools found\n")
+	default:
+		ctx.Printf("%s", renderServiceHealth(health))
+	}
+
 	// Recent deployments summary
 	ctx.Printf("\n%s\n", labelStyle.Render("Recent Activity:"))
 
@@ -288,7 +299,7 @@ func printAppStatusJSON(
 	appConfig *app_v1alpha.Configuration,
 	activeDeployment *deployment_v1alpha.DeploymentClientGetActiveDeploymentResults,
 	recentResult *deployment_v1alpha.DeploymentClientListDeploymentsResults,
-	app, cluster string,
+	app, cluster string, health []serviceHealth, healthErr error,
 ) error {
 	type gitInfo struct {
 		Sha               string `json:"sha,omitempty"`
@@ -396,11 +407,17 @@ func printAppStatusJSON(
 		Configuration     *configuration   `json:"configuration,omitempty"`
 		ActiveDeployment  *deploymentJSON  `json:"active_deployment,omitempty"`
 		RecentDeployments []deploymentJSON `json:"recent_deployments,omitempty"`
+		Services          []serviceHealth  `json:"services,omitempty"`
+		HealthError       string           `json:"health_error,omitempty"`
 	}{
 		App:               app,
 		Cluster:           cluster,
 		WorkloadRole:      appResult.WorkloadRole(),
 		MaintenanceRoutes: appResult.MaintenanceRoutes(),
+		Services:          health,
+	}
+	if healthErr != nil {
+		output.HealthError = healthErr.Error()
 	}
 
 	if appResult.HasVersionId() && appResult.VersionId() != "" {