[Spec] Reconcile hydrated state against reality on rad startup

pkg/cli/cmd/startup/startup.go

@@ -175,6 +175,17 @@ func (r *Runner) Run(ctx context.Context) error {
 	}
 	scaledBackUp = true
 
+	// ReconcileHydratedState is best-effort: it POSTs the reconcile custom action per application
+	// so the state store reflects reality before the next 'rad ...' command runs. Failures are
+	// logged and the workflow still succeeds; see specs/006-state-restoration.
+	r.Output.LogInfo("Reconciling hydrated state against reality...")
+	reports, err := r.StateClient.ReconcileHydratedState(ctx, r.Workspace)
+	if err != nil {
+		r.Output.LogInfo("Reconcile skipped: %v", err)
+	} else {
+		logReconcileReports(r.Output, reports)
+	}
+
 	r.Output.LogInfo("State restored successfully.")
 	return nil
 }

pkg/cli/cmd/startup/startup_test.go

@@ -76,10 +76,15 @@ type fakeStateRestoreClient struct {
 	waitErr      error
 	restoreDBErr error
 	restoreTFErr error
+	reconcileErr error
 
-	waited   bool
-	dbCalled bool
-	tfCalled bool
+	waited          bool
+	dbCalled        bool
+	tfCalled        bool
+	reconcileCalled bool
+
+	reconcileReports []ApplicationReconcileReport
+	reconcileArg     *workspaces.Workspace
 
 	order []string
 }
@@ -102,6 +107,16 @@ func (f *fakeStateRestoreClient) RestoreTerraform(ctx context.Context, kubeConte
 	return f.restoreTFErr
 }
 
+func (f *fakeStateRestoreClient) ReconcileHydratedState(ctx context.Context, workspace *workspaces.Workspace) ([]ApplicationReconcileReport, error) {
+	f.reconcileCalled = true
+	f.reconcileArg = workspace
+	f.order = append(f.order, "reconcile")
+	if f.reconcileErr != nil {
+		return nil, f.reconcileErr
+	}
+	return f.reconcileReports, nil
+}
+
 // fakeScaler records scale operations and appends them to a shared order slice so tests can assert
 // that the control plane is scaled down before any restore and back up afterward.
 type fakeScaler struct {
@@ -177,10 +192,46 @@ func Test_Run_RestoresInOrderWaitDatabaseTerraform(t *testing.T) {
 	require.True(t, client.waited)
 	require.True(t, client.dbCalled)
 	require.True(t, client.tfCalled)
+	require.True(t, client.reconcileCalled)
 	require.True(t, scaler.downCalled)
 	require.True(t, scaler.upCalled)
-	require.Equal(t, []string{"scaledown", "wait", "db", "tf", "scaleup"}, client.order,
-		"must scale down, wait, restore databases, restore terraform, then scale up")
+	require.Equal(t, []string{"scaledown", "wait", "db", "tf", "scaleup", "reconcile"}, client.order,
+		"reconcile must run after scale up so the resource providers are ready to serve it")
+}
+
+// Test_Run_ReconcileFailureDoesNotFailStartup verifies the best-effort contract of the reconcile
+// stage: when ReconcileHydratedState returns an error, rad startup logs and still succeeds. Test
+// case matches the spec's acceptance criterion "rad startup never fails because reconciliation
+// could not reach a resource provider".
+func Test_Run_ReconcileFailureDoesNotFailStartup(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	client := &fakeStateRestoreClient{reconcileErr: errors.New("ucp unreachable")}
+	r, _ := newTestRunner(t, ctrl, client)
+
+	err := r.Run(t.Context())
+	require.NoError(t, err, "reconcile failure must not fail rad startup")
+	require.True(t, client.reconcileCalled)
+}
+
+// Test_Run_ReconcileReceivesWorkspace verifies that the workspace passed to the reconcile stage
+// is the runner's active workspace, so the default client can build a connection to the right
+// control plane.
+func Test_Run_ReconcileReceivesWorkspace(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	client := &fakeStateRestoreClient{
+		reconcileReports: []ApplicationReconcileReport{
+			{Name: "cool-app", ResourceCount: 3},
+		},
+	}
+	r, _ := newTestRunner(t, ctrl, client)
+
+	require.NoError(t, r.Run(t.Context()))
+	require.NotNil(t, client.reconcileArg)
+	require.Equal(t, r.Workspace, client.reconcileArg)
 }
 
 func Test_Run_ScaleDownFailureStopsBeforeRestore(t *testing.T) {

pkg/cli/cmd/startup/stateclient.go

@@ -18,10 +18,17 @@ package startup
 
 import (
 	"context"
+	"errors"
+	"fmt"
 
+	"github.com/radius-project/radius/pkg/azure/tokencredentials"
 	"github.com/radius-project/radius/pkg/cli/controlplane"
+	"github.com/radius-project/radius/pkg/cli/output"
 	"github.com/radius-project/radius/pkg/cli/pgbackup"
 	"github.com/radius-project/radius/pkg/cli/tfstate"
+	"github.com/radius-project/radius/pkg/cli/workspaces"
+	corerpv20250801preview "github.com/radius-project/radius/pkg/corerp/api/v20250801preview"
+	"github.com/radius-project/radius/pkg/sdk"
 )
 
 // ControlPlaneScaler scales the database-backed control-plane deployments to zero and back, so
@@ -53,6 +60,31 @@ type StateRestoreClient interface {
 
 	// RestoreTerraform re-creates the Terraform state Secrets from stateDir.
 	RestoreTerraform(ctx context.Context, kubeContext, namespace, stateDir string) error
+
+	// ReconcileHydratedState invokes the reconcile custom action on every application in the
+	// workspace's plane. It is best-effort: individual per-application failures are recorded in
+	// the returned reports but never propagate as a fatal error. Called by 'rad startup' after
+	// ScaleUp so subsequent commands see reality-checked state.
+	//
+	// A non-nil error is returned only when the pass could not begin at all (for example, the
+	// workspace's control plane is unreachable). Callers should treat such errors as advisory and
+	// still return success from the outer startup command.
+	ReconcileHydratedState(ctx context.Context, workspace *workspaces.Workspace) ([]ApplicationReconcileReport, error)
+}
+
+// ApplicationReconcileReport captures the per-application outcome of ReconcileHydratedState. One
+// entry is produced for every application the reconcile pass attempted, whether it succeeded or
+// not.
+type ApplicationReconcileReport struct {
+	// Name is the application resource name (not the fully-qualified resource ID).
+	Name string
+	// ResourceCount is the number of child resources the reconcile handler reported an outcome
+	// for. Zero when the reconcile handler is still a stub, when the application has no
+	// non-terminal children, or when the reconcile call itself failed.
+	ResourceCount int
+	// Err is set when the reconcile call for this application failed. The pass continues to the
+	// next application regardless.
+	Err error
 }
 
 // defaultStateRestoreClient is the production implementation.
@@ -78,3 +110,81 @@ func (defaultStateRestoreClient) RestoreTerraform(ctx context.Context, kubeConte
 	}
 	return client.Restore(ctx, stateDir)
 }
+
+// ReconcileHydratedState lists every application in the workspace's plane and POSTs the
+// Radius.Core/applications 'reconcile' custom action on each. Reports are aggregated across
+// pagination and returned to the caller.
+func (defaultStateRestoreClient) ReconcileHydratedState(ctx context.Context, workspace *workspaces.Workspace) ([]ApplicationReconcileReport, error) {
+	if workspace == nil {
+		return nil, errors.New("workspace is required")
+	}
+
+	connection, err := workspace.Connect(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to workspace: %w", err)
+	}
+
+	clientOptions := sdk.NewClientOptions(connection)
+	factory, err := corerpv20250801preview.NewClientFactory(&tokencredentials.AnonymousCredential{}, clientOptions)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build Radius.Core client factory: %w", err)
+	}
+	applications := factory.NewApplicationsClient()
+
+	// Collect application names first so a stalled reconcile does not stall the LIST.
+	names, err := listApplicationNames(ctx, applications, workspace.Scope)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list applications for reconcile: %w", err)
+	}
+
+	reports := make([]ApplicationReconcileReport, 0, len(names))
+	for _, name := range names {
+		report := ApplicationReconcileReport{Name: name}
+		resp, err := applications.Reconcile(ctx, workspace.Scope, name, corerpv20250801preview.ReconcileRequest{}, nil)
+		if err != nil {
+			report.Err = err
+		} else {
+			report.ResourceCount = len(resp.Resources)
+		}
+		reports = append(reports, report)
+	}
+	return reports, nil
+}
+
+// listApplicationNames walks the paginated ListByScope response for `scope` and returns the
+// application resource names. Nil entries and entries without a Name are skipped.
+func listApplicationNames(ctx context.Context, client *corerpv20250801preview.ApplicationsClient, scope string) ([]string, error) {
+	pager := client.NewListByScopePager(scope, &corerpv20250801preview.ApplicationsClientListByScopeOptions{})
+	var names []string
+	for pager.More() {
+		page, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, err
+		}
+		for _, app := range page.Value {
+			if app == nil || app.Name == nil {
+				continue
+			}
+			names = append(names, *app.Name)
+		}
+	}
+	return names, nil
+}
+
+// logReconcileReports emits one line per application, and a summary line if any application
+// failed. Used by rad startup's ReconcileHydratedState stage to surface outcomes in the workflow
+// log.
+func logReconcileReports(out output.Interface, reports []ApplicationReconcileReport) {
+	failed := 0
+	for _, r := range reports {
+		if r.Err != nil {
+			failed++
+			out.LogInfo("  reconcile %s: failed (%s)", r.Name, r.Err.Error())
+			continue
+		}
+		out.LogInfo("  reconcile %s: reconciled %d resource(s)", r.Name, r.ResourceCount)
+	}
+	if failed > 0 {
+		out.LogInfo("Reconcile completed with %d/%d application failures; continuing.", failed, len(reports))
+	}
+}