From 6e621a77c01d624ff20ceb816bdb658875c8abc9 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:36:24 +0100 Subject: [PATCH 01/18] Reference resilient Azure.AI.AgentServer preview packages Point the AgentServer package versions at the drop that carries the resilient / checkpointing / steering surface (Core beta.27, Invocations beta.6, Responses beta.7) and add the agentserver-preview-local source mapped to Azure.AI.AgentServer.*. --- dotnet/Directory.Packages.props | 6 +++--- dotnet/nuget.config | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 73a7ab8bf25..ecebc889c9a 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -23,9 +23,9 @@ - - - + + + diff --git a/dotnet/nuget.config b/dotnet/nuget.config index 128d95e590c..3cde656b2a4 100644 --- a/dotnet/nuget.config +++ b/dotnet/nuget.config @@ -3,10 +3,14 @@ + + + + From 39e4d531e1401bf5073b57b2dc7b752bd7343c33 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:36:35 +0100 Subject: [PATCH 02/18] Add ADR 0032 for resilient long-running Foundry hosting Document, in plain language, when durable long-running hosting applies (background requests only; workflow or safe-to-redo single agent), the recovery contract, the resilience gate, host-wide single registration, the agent-registration readiness health check, session-store durability, and the two focused samples. --- ...y-hosting-resilient-long-running-agents.md | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md diff --git a/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md b/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md new file mode 100644 index 00000000000..5c67b0f423b --- /dev/null +++ b/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md @@ -0,0 +1,176 @@ +--- +status: proposed +contact: rogerbarreto +date: 2026-07-17 +deciders: Roger Barreto, Ben Thomas +consulted: Tao Chen, Ravi Teja Pidaparthi, Glenn Condron +informed: Agent Framework .NET team +--- + +# Resilient / durable long-running agents in Microsoft.Agents.AI.Foundry.Hosting + +## Context and Problem Statement + +The Foundry Hosted Agents platform can now run a hosted agent as a long job that keeps going even +when no client is connected, and that the platform restarts on its own after the container crashes +or is recycled. When the platform restarts the agent after a crash, it calls the handler again with +the same original input, tells the handler this is a restart (a flag called `IsRecovery`), and hands +back the last saved copy of the response so far (a property called `PersistedResponse`). + +This behavior only applies to **background** requests. A background request is one where the caller +starts the work and checks back later, instead of holding the connection open and waiting. For a +normal foreground request (the caller waits on the line), there is no durability: if the server +crashes, the request simply fails. + +The Python Microsoft Agent Framework already has a reference implementation of this, authored by Tao +Chen. A workflow (a graph of steps that saves its own progress as it advances) is hosted with two +switches turned on: one that asks the platform for crash-restart plus replay of already-sent output +(`resilient_background`), and one that lets a caller send a new message in the middle of a running +turn (`steerable_conversations`). + +We want the same capability in the .NET `Microsoft.Agents.AI.Foundry.Hosting` package, so a Microsoft +Agent Framework workflow hosted as an `AIAgent` can run as a durable, crash-recoverable, and +optionally steerable Foundry Hosted Agent. + +The platform primitives are provided by the Azure `Azure.AI.AgentServer.*` packages (Core +`1.0.0-beta.27`, Responses `1.0.0-beta.7`, Invocations `1.0.0-beta.6`), which expose the resilience, +recovery, and steering surface used here. + +## Decision Drivers + +- Match the Python reference behavior: the same recovery contract and the same opt-in shape. +- Everything is opt-in and off by default. An application that does not ask for resilience must + behave exactly as it does today and must pay no extra cost. +- Do the durable work where it makes sense. A workflow already saves its own progress, so it can be + reloaded and continued after a crash. A single agent keeps nothing in the middle of a turn, so the + best it can do is redo the whole turn. +- Reuse what already exists. The `AgentFrameworkResponseHandler` already implements the streaming + `CreateAsync(CreateResponse, ResponseContext, ...)` method that recovery uses, and workflows are + already hosted as agents through `WorkflowBuilder(...).Build().AsAIAgent(name)`. +- Keep the public API lean. Prefer the Azure SDK types directly over wrappers that only duplicate + them. + +## Considered Options + +- **A. Turn resilience on through the existing handler, workflow first.** Let the registration method + flow the two switches straight to the Azure SDK options type (`ResponsesServerOptions`), and connect + the workflow's own saved progress to the platform's crash-restart path + (`ResponseContext.IsRecovery` / `PersistedResponse` / `ExitForRecoveryAsync`, and + `ResponseEventStream.Checkpoint()` to save a snapshot at a step boundary). +- **B. A separate durable-hosting package or handler.** A new package parallel to the existing one. + +## Decision Outcome + +Chosen option: **A**. It matches the Python behavior, reuses the handler and the workflow-hosting path +already in place, and keeps the change additive and opt-in (off by default, identical to today's +behavior unless a switch is turned on). Option B would duplicate the handler and add a parallel +hosting path for no functional gain. + +### The recovery contract (same as Python) + +The platform saves the inputs but not the outputs. After a crash it calls the handler again with the +same input and with `IsRecovery` set to true. The handler rebuilds where it left off from the saved +snapshot only (work that was in flight and not yet saved is left out), and emits an early +`in_progress` event that the client sees as a reset point. On a graceful shutdown the handler calls +`ExitForRecoveryAsync()` to stop cleanly so the platform can restart it later. + +### How the handler knows resilience is on (the gate) + +The request context does not carry a "resilience is on" flag. The handler figures it out from two +sources: + +- **Recovery branch (reload after a crash):** the only condition is `context.IsRecovery`. That flag + is true only when resilience is on and a crash happened, so on a normal run the handler never + touches the recovery logic and pays nothing. +- **Save-progress branch (during the first run):** the condition is resilience-on **and** + `request.Background` **and** `request.Store`. Only when all three are true is it worth saving + progress. The handler reads whether resilience is on by injecting `IOptions` + and reading `ResilientBackground`. Saving a snapshot is already a no-op when the response is not a + resilient background response, so an accidental call is harmless; the gate just avoids the wasted + work of building a snapshot that would be thrown away. + +### Resilience is configured once per process, never per agent + +The Responses server (and its resilience setting) is one per process. Registering it twice throws. +Multiple agents are hosted by registering each agent as a keyed service and calling the parameterless +registration method once; they all share the one server configuration. Because of this: + +- The `AddFoundryResponses(agent)` overload is a shortcut for a single agent. If the server is already + registered, it fails with a clear message telling the developer to use the parameterless + `AddFoundryResponses()` and register each agent with `AddKeyedSingleton`. This prevents + mixing the single-agent shortcut with more agents and prevents a confusing double registration. +- The parameterless `AddFoundryResponses()` is the path for one or many keyed agents and builds the + server once. + +### Fail early when no agent is registered + +Today, if no agent is registered, the failure only appears on the first request. We surface it earlier +through the readiness probe (`/readiness`), which the platform checks before sending any traffic. A +health check named for agent registration reports healthy when at least one `AIAgent` is registered +(keyed or default) and unhealthy with an actionable message when none is. This follows the same +pattern already used for the toolbox health check. + +### Where the workflow's saved progress lives + +A new `FoundryResilientAgentSessionStore` (another implementation of the existing `AgentSessionStore` +base class) owns the resilience-specific persistence: the small pointer to the workflow's last saved +progress per conversation, kept in the platform's per-conversation metadata +(`ConversationChainMetadata`), while the larger workflow state stays in durable storage. The default +store is chosen by configuration: when resilience is on, the registration uses this resilient store; +when resilience is off, it keeps the current `FileSystemAgentSessionStore`. The choice is made lazily +in the store factory, which reads `IOptions` at host startup. A store supplied +explicitly by the developer is always respected. + +### Two focused samples + +One sample demonstrates resilience only (start a long background job, crash the process in the middle, +watch it restart and continue from its last saved progress). A second sample demonstrates steering +only (send a new message while a turn is running and watch it queue and resume at a safe point). +Keeping each sample focused on one behavior makes it easier to demonstrate and to follow. + +### Applicability to Microsoft Agent Framework hosting + +- **Workflow hosted as an agent (full support):** the workflow reloads its own saved progress after a + crash and continues, so at most the step that was interrupted repeats. +- **Single agent (best-effort, with documented limits):** a single agent keeps nothing in the middle + of a turn, so after a crash it redoes the whole turn from the start. This is correct as long as the + turn is safe to run again. If the turn performs an action that must not happen twice (for example + sending an email or charging a card), a small guard is needed so the redo does not repeat it. +- **Do not bother** for quick foreground chat requests: there is no durability in that mode, so the + switches change nothing. + +## Implementation plan + +- **Done:** reference the `Azure.AI.AgentServer.*` versions that carry the resilient surface + (`Directory.Packages.props`), add the local package source (`dotnet/nuget.config`). +- **Done:** lean opt-in surface. `AddFoundryResponses(...)` accepts `Action?` + directly and passes it to `AddResponsesServer(...)`. There is no wrapper options type: the Azure SDK + `ResponsesServerOptions` already exposes `ResilientBackground` and `SteerableConversations` (along + with `DefaultModel`, `DefaultFetchHistoryCount`, and `ResponseAcceptor`). +- **Planned:** the resilience gate, the single-registration guard, and the agent-registration health + check on the registration method. +- **Planned:** the `FoundryResilientAgentSessionStore` with configuration-driven selection. +- **Planned:** the workflow recovery bridge in `AgentFrameworkResponseHandler.CreateAsync`. On + `IsRecovery`, reload the workflow from its last saved progress and continue; save a snapshot at each + step boundary; call `ExitForRecoveryAsync()` on shutdown. +- **Planned:** single-agent best-effort behavior plus a guard hook for actions that must not repeat. +- **Planned:** the two focused samples, each runnable locally with crash-and-recover or steering. + +## Consequences + +- Good: durable long-running agents on .NET with the same developer experience as Python. The change + is additive and off by default, so existing hosted agents are unaffected unless they opt in. +- Good: the public API stays lean by using the Azure SDK options type directly. +- Follow-up: keep the `Azure.AI.AgentServer.*` version pins tracking the resilient surface as it + evolves. + +## Python to .NET mapping + +| Python (reference) | .NET | +| --- | --- | +| `ResponsesServerOptions(resilient_background=True)` | `AddFoundryResponses(o => o.ResilientBackground = true)` on `ResponsesServerOptions` | +| `steerable_conversations=True` | `o.SteerableConversations = true` on `ResponsesServerOptions` | +| `context.is_recovery` / `context.persisted_response` | `ResponseContext.IsRecovery` / `PersistedResponse` | +| `await context.exit_for_recovery()` | `ResponseContext.ExitForRecoveryAsync()` | +| `yield stream.checkpoint()` | `ResponseEventStream.Checkpoint()` | +| `Workflow.as_agent(...)` + saved workflow progress | `WorkflowBuilder(...).Build().AsAIAgent(...)` + `FoundryResilientAgentSessionStore` | From dbf53fc92ef6e19cfb5e20a03a70f19a9be9cd7b Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:36:56 +0100 Subject: [PATCH 03/18] Add resilient hosting opt-in surface and workflow save cadence Lean opt-in wiring and the crash-recovery save cadence, all inert unless resilience is turned on: - AddFoundryResponses flows Action straight to the SDK (no wrapper options type). - Single-registration guard: the Responses server is host-wide (one per process). A prior registration is detected via the ResponseHandler it always registers; the single-agent AddFoundryResponses(agent) extension throws an actionable message if combined or repeated, and the parameterless overload is a no-op on repeat. - Agent-registration readiness health check: unhealthy until at least one AIAgent is registered (keyed or default), scanning service descriptors so it never constructs an agent. - Resilience gate: the handler reads IOptions for ResilientBackground; ShouldPersistForResilience requires background and store. - Save cadence: on a resilient turn (or IsRecovery), persist the session at each completed output item so a mid-turn crash leaves the last workflow step on disk; on graceful shutdown, ExitForRecoveryAsync instead of emitting incomplete. - Tests for the guard and the health check. --- .../AgentFrameworkResponseHandler.cs | 59 ++++++++++++- .../AgentRegistrationHealthCheck.cs | 48 +++++++++++ .../ServiceCollectionExtensions.cs | 86 ++++++++++++++++++- .../AgentRegistrationHealthCheckTests.cs | 66 ++++++++++++++ .../ServiceCollectionExtensionsTests.cs | 68 ++++++++++++++- 5 files changed, 320 insertions(+), 7 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentRegistrationHealthCheck.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentRegistrationHealthCheckTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index 538d424ad4f..d31182d8966 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.Shared.DiagnosticIds; namespace Microsoft.Agents.AI.Foundry.Hosting; @@ -27,6 +28,14 @@ public class AgentFrameworkResponseHandler : ResponseHandler private readonly ILogger _logger; private readonly FoundryToolboxService? _toolboxService; + /// + /// Whether the host-wide Responses server was configured for durable long-running (resilient) + /// background responses (). This is the + /// first half of the resilience gate: when it is the handler never does + /// any checkpoint/recovery work and behaves exactly as a non-resilient host. + /// + private readonly bool _resilientBackground; + /// /// Cached fallback used when no is registered in DI. /// Avoids a per-request allocation on the request hot path. @@ -40,10 +49,16 @@ public class AgentFrameworkResponseHandler : ResponseHandler /// The service provider for resolving agents. /// The logger instance. /// Optional Foundry Toolbox service providing MCP tools. + /// + /// The host-wide Responses server options, used only to read whether resilient background + /// responses are enabled. Optional so the handler can be constructed without the options + /// registered (for example in unit tests), in which case resilience is treated as off. + /// public AgentFrameworkResponseHandler( IServiceProvider serviceProvider, ILogger logger, - FoundryToolboxService? toolboxService = null) + FoundryToolboxService? toolboxService = null, + IOptions? responsesServerOptions = null) { ArgumentNullException.ThrowIfNull(serviceProvider); ArgumentNullException.ThrowIfNull(logger); @@ -51,8 +66,18 @@ public AgentFrameworkResponseHandler( this._serviceProvider = serviceProvider; this._logger = logger; this._toolboxService = toolboxService; + this._resilientBackground = responsesServerOptions?.Value.ResilientBackground ?? false; } + /// + /// The resilience gate for the save-progress path: checkpoint work is worthwhile only when the + /// host enabled resilient background responses and this specific request is a stored background + /// response. When any part is false, the request runs exactly as it does on a non-resilient host + /// (the recovery path is gated separately on ResponseContext.IsRecovery). + /// + private bool ShouldPersistForResilience(CreateResponse request) + => this._resilientBackground && request.Background == true && request.Store == true; + /// public override async IAsyncEnumerable CreateAsync( CreateResponse request, @@ -349,6 +374,15 @@ await this._toolboxService // 7. Run the agent and convert output // NOTE: C# forbids 'yield return' inside a try block that has a catch clause, // and inside catch blocks. We use a flag to defer the yield to outside the try/catch. + // + // Resilient turn: when the host enabled resilient background responses for this stored + // background request (or this is a crash-recovery re-entry), the session is persisted at + // each output-item boundary so a mid-turn crash leaves the last completed workflow superstep + // on disk. A workflow hosted as an agent already checkpoints per superstep and resumes from + // its restored checkpoint on load; the only gap is that the session was previously persisted + // only at end-of-turn. All of this is gated: a non-resilient host runs exactly as before. + bool isResilientTurn = this.ShouldPersistForResilience(request) || context.IsRecovery; + bool emittedTerminal = false; var enumerator = OutputConverter.ConvertUpdatesToEventsAsync( agent.RunStreamingAsync(messages, session, options: options, cancellationToken: consentCts.Token), @@ -424,7 +458,17 @@ await this._toolboxService if (shutdownDetected) { - // Server is shutting down — emit incomplete so clients can resume + // Server is shutting down. On a resilient turn, defer for recovery instead of + // failing: leaving the response in_progress lets the platform re-invoke the + // handler next lifetime, where the workflow resumes from its last persisted + // superstep. On a non-resilient turn, keep today's behavior (emit incomplete). + if (isResilientTurn) + { + this._logger.LogInformation("Shutdown detected on a resilient turn; deferring for recovery."); + await context.ExitForRecoveryAsync(cancellationToken).ConfigureAwait(false); + yield break; + } + this._logger.LogInformation("Shutdown detected, emitting incomplete response."); yield return stream.EmitIncomplete(); yield break; @@ -433,6 +477,17 @@ await this._toolboxService // yield is in the outer try (finally-only) — allowed by C# yield return evt!; + // Resilient checkpoint cadence: persist the session at each completed output item, + // a natural workflow phase boundary. Idempotent and gated; awaiting here is safe + // because this sits in the outer (finally-only) try, not the inner try/catch. + if (isResilientTurn + && evt is ResponseOutputItemDoneEvent + && session is not null + && !string.IsNullOrWhiteSpace(sessionConversationId)) + { + await sessionStore.SaveSessionAsync(agent, sessionConversationId, session, resolvedUserId, cancellationToken).ConfigureAwait(false); + } + if (evt is ResponseCompletedEvent or ResponseFailedEvent or ResponseIncompleteEvent) { emittedTerminal = true; diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentRegistrationHealthCheck.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentRegistrationHealthCheck.cs new file mode 100644 index 00000000000..199f30b100e --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentRegistrationHealthCheck.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI.Foundry.Hosting; + +/// +/// Readiness health check that reports unhealthy until at least one is +/// registered (as the default service or under any key). It is mapped onto the /readiness +/// probe by , which the Foundry platform +/// calls before routing any request. Surfacing a missing-agent misconfiguration here turns what +/// would otherwise be a per-request failure (the handler cannot resolve an agent) into an early, +/// actionable not-ready signal. +/// +/// +/// The check runs a predicate over the registered service descriptors, so it detects a keyed or +/// default registration without resolving (and therefore without constructing) any agent. It has +/// no side effects and is cheap to run on every readiness poll. +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +internal sealed class AgentRegistrationHealthCheck : IHealthCheck +{ + private readonly Func _hasAnyAgent; + + public AgentRegistrationHealthCheck(Func hasAnyAgent) + { + ArgumentNullException.ThrowIfNull(hasAnyAgent); + this._hasAnyAgent = hasAnyAgent; + } + + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + if (this._hasAnyAgent()) + { + return Task.FromResult(HealthCheckResult.Healthy( + description: "Foundry agents: at least one AIAgent is registered.")); + } + + return Task.FromResult(new HealthCheckResult( + status: context.Registration.FailureStatus, + description: "Foundry agents: no AIAgent is registered. Register an agent via AddFoundryResponses(agent), or services.AddKeyedSingleton(name, agent) followed by AddFoundryResponses().")); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs index 8e2a6986f92..fbd1ba3b00e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs @@ -48,11 +48,15 @@ public static class FoundryHostingExtensions /// /// /// The service collection. + /// + /// Optional callback to configure the underlying , for example to opt in to + /// durable long-running (resilient) execution via . + /// /// The service collection for chaining. - public static IServiceCollection AddFoundryResponses(this IServiceCollection services) + public static IServiceCollection AddFoundryResponses(this IServiceCollection services, Action? configure = null) { ArgumentNullException.ThrowIfNull(services); - services.AddResponsesServer(); + AddFoundryResponsesServer(services, configure, singleAgentExtension: false); services.AddHealthChecks(); services.TryAddSingleton(_ => FileSystemAgentSessionStore.CreateDefault()); services.TryAddSingleton(); @@ -82,13 +86,17 @@ public static IServiceCollection AddFoundryResponses(this IServiceCollection ser /// The service collection. /// The agent instance to register. /// The agent session store to use for managing agent sessions server-side. If null, a file-system session store is used, rooted at /.checkpoints when running in a Foundry hosted environment and {cwd}/.checkpoints locally. + /// + /// Optional callback to configure the underlying , for example to opt in to + /// durable long-running (resilient) execution via . + /// /// The service collection for chaining. - public static IServiceCollection AddFoundryResponses(this IServiceCollection services, AIAgent agent, AgentSessionStore? agentSessionStore = null) + public static IServiceCollection AddFoundryResponses(this IServiceCollection services, AIAgent agent, AgentSessionStore? agentSessionStore = null, Action? configure = null) { ArgumentNullException.ThrowIfNull(services); ArgumentNullException.ThrowIfNull(agent); - services.AddResponsesServer(); + AddFoundryResponsesServer(services, configure, singleAgentExtension: true); services.AddHealthChecks(); agentSessionStore ??= FileSystemAgentSessionStore.CreateDefault(); @@ -276,6 +284,76 @@ private static void MapReadinessIfMissing(IEndpointRouteBuilder endpoints) endpoints.MapHealthChecks(ReadinessPath); } + /// + /// Registers the Azure AI Responses server exactly once, applying the optional + /// configuration callback, and registers the + /// agent-registration readiness health check. + /// + /// + /// The Responses server (and its resilience configuration) is host-wide: exactly one per + /// process. The underlying AddResponsesServer registers a resilient task under a fixed + /// name and throws if invoked twice. A prior AddFoundryResponses call is detected by the + /// it always registers, so a second call is handled here with an + /// actionable message instead of throwing deep inside the SDK. + /// + /// The service collection. + /// Optional configuration callback. + /// + /// when called from the single-agent AddFoundryResponses(agent) + /// extension; a repeat registration on this path is a usage error and throws. + /// + private static void AddFoundryResponsesServer(IServiceCollection services, Action? configure, bool singleAgentExtension) + { + // Both AddFoundryResponses overloads register a ResponseHandler, so its presence means the + // host-wide server was already set up by a previous call. + var alreadyRegistered = services.Any(d => d.ServiceType == typeof(ResponseHandler)); + if (alreadyRegistered) + { + if (singleAgentExtension) + { + throw new InvalidOperationException( + "A Foundry Responses server is already registered. AddFoundryResponses(agent) is a single-agent extension and cannot be combined with, or repeated alongside, another AddFoundryResponses call. To host multiple agents, call AddFoundryResponses() once and register each agent with services.AddKeyedSingleton(name, agent)."); + } + + // Parameterless path: the host-wide server is already configured. Registering more keyed + // agents does not require (and must not repeat) the server registration, so this is a no-op. + return; + } + + services.AddResponsesServer(configure ?? (_ => { })); + AddAgentRegistrationHealthCheck(services); + } + + /// + /// Registers a /readiness health check that reports unhealthy until at least one + /// is registered (keyed or default). Because the platform probes + /// /readiness before routing any request, this surfaces a missing-agent misconfiguration + /// at startup instead of failing the first invocation. Mirrors the toolbox health-check + /// registration and dedupes by name so a host that already added it is not double-registered. + /// + private static void AddAgentRegistrationHealthCheck(IServiceCollection services) + { + const string HealthCheckName = "foundry-agent-registration"; + services.AddHealthChecks(); + services.Configure(opts => + { + foreach (var existing in opts.Registrations) + { + if (string.Equals(existing.Name, HealthCheckName, StringComparison.Ordinal)) + { + return; + } + } + + opts.Registrations.Add(new HealthCheckRegistration( + name: HealthCheckName, + factory: _ => new AgentRegistrationHealthCheck( + () => services.Any(d => d.ServiceType == typeof(AIAgent))), + failureStatus: HealthStatus.Unhealthy, + tags: ["foundry", "agent", "readiness"])); + }); + } + /// /// The ActivitySource name for the Responses hosting pipeline. /// diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentRegistrationHealthCheckTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentRegistrationHealthCheckTests.cs new file mode 100644 index 00000000000..7293bc80393 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentRegistrationHealthCheckTests.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Moq; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +public class AgentRegistrationHealthCheckTests +{ + [Fact] + public async Task CheckHealthAsync_AgentPresent_ReturnsHealthyAsync() + { + // Arrange: the predicate reports an agent is registered (default or keyed). + var check = new AgentRegistrationHealthCheck(() => true); + + // Act + var result = await check.CheckHealthAsync(NewContext(HealthStatus.Unhealthy)); + + // Assert + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + [Fact] + public async Task CheckHealthAsync_NoAgent_ReturnsConfiguredFailureAsync() + { + // Arrange: the predicate reports no agent registered. + var check = new AgentRegistrationHealthCheck(() => false); + + // Act + var result = await check.CheckHealthAsync(NewContext(HealthStatus.Unhealthy)); + + // Assert + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("no AIAgent is registered", result.Description, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void DescriptorPredicate_DetectsKeyedAndDefaultAgents() + { + // Guards the descriptor-based detection the registration wires in: both a default and a + // keyed AIAgent registration expose ServiceType == typeof(AIAgent). + var keyed = new ServiceCollection(); + keyed.AddKeyedSingleton("my-agent", new Mock().Object); + Assert.Contains(keyed, d => d.ServiceType == typeof(AIAgent)); + + var byDefault = new ServiceCollection(); + byDefault.AddSingleton(new Mock().Object); + Assert.Contains(byDefault, d => d.ServiceType == typeof(AIAgent)); + + var empty = new ServiceCollection(); + Assert.DoesNotContain(empty, d => d.ServiceType == typeof(AIAgent)); + } + + private static HealthCheckContext NewContext(HealthStatus failureStatus) => + new() + { + Registration = new HealthCheckRegistration( + name: "foundry-agent-registration", + instance: Mock.Of(), + failureStatus: failureStatus, + tags: null), + }; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs index f49c669f66d..0e0a484dc62 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs @@ -45,6 +45,70 @@ public void AddFoundryResponses_CalledTwice_RegistersOnce() Assert.Equal(1, count); } + [Fact] + public void AddFoundryResponses_WithAgent_CalledTwice_ThrowsActionable() + { + // Arrange: the single-agent extension is not composable; a second call must be rejected + // with guidance to use the parameterless overload plus keyed agents for multiple agents. + var services = new ServiceCollection(); + services.AddLogging(); + var agent = new Mock().Object; + + services.AddFoundryResponses(agent); + + // Act + Assert + var ex = Assert.Throws(() => services.AddFoundryResponses(agent)); + Assert.Contains("single-agent extension", ex.Message, StringComparison.OrdinalIgnoreCase); + Assert.Contains("AddKeyedSingleton", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void AddFoundryResponses_ParameterlessThenWithAgent_Throws() + { + // Arrange: once the multi-agent path built the host-wide server, the single-agent extension + // cannot be added on top of it. + var services = new ServiceCollection(); + services.AddLogging(); + + services.AddFoundryResponses(); + + // Act + Assert + Assert.Throws( + () => services.AddFoundryResponses(new Mock().Object)); + } + + [Fact] + public void AddFoundryResponses_WithAgentThenParameterless_DoesNotThrow() + { + // Arrange: the single-agent extension builds the server; a later parameterless call (for + // example from shared wiring) must be a safe no-op rather than a throw. + var services = new ServiceCollection(); + services.AddLogging(); + services.AddFoundryResponses(new Mock().Object); + + // Act + var exception = Record.Exception(() => services.AddFoundryResponses()); + + // Assert + Assert.Null(exception); + Assert.Equal(1, services.Count(d => d.ServiceType == typeof(ResponseHandler))); + } + + [Fact] + public void AddFoundryResponses_RegistersAgentRegistrationHealthCheck() + { + var services = new ServiceCollection(); + services.AddLogging(); + + services.AddFoundryResponses(); + + var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService< + Microsoft.Extensions.Options.IOptions< + Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckServiceOptions>>(); + Assert.Contains(options.Value.Registrations, r => r.Name == "foundry-agent-registration"); + } + [Fact] public void AddFoundryResponses_NullServices_ThrowsArgumentNullException() { @@ -74,8 +138,10 @@ public void AddFoundryResponses_WithAgent_RegistersAgentAndHandler() public void AddFoundryResponses_WithNullAgent_ThrowsArgumentNullException() { var services = new ServiceCollection(); + // Cast to bind the agent overload: the parameterless overload also accepts a single null + // (as its optional configure callback), so the cast keeps this test targeting the agent path. Assert.Throws( - () => services.AddFoundryResponses(null!)); + () => services.AddFoundryResponses((AIAgent)null!)); } [Fact] From c2efac5e59de0044b8016ac1696fd44f4909d935 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:37:11 +0100 Subject: [PATCH 04/18] Add resilient and steering hosting samples - Hosted-Workflow-Resilient: the translation-chain workflow hosted with ResilientBackground enabled, so a background response survives a container crash and resumes from the workflow's last completed step. README has a local crash-and-recover walkthrough. - Hosted-Steering: a chat agent hosted with SteerableConversations enabled, so a follow-up input for an in-progress conversation is queued instead of rejected. Both opt in with a single option (o => o.ResilientBackground/SteerableConversations = true), are off by default, registered in agent-framework-dotnet.slnx, build clean, and answer GET /readiness with 200 Healthy locally. --- dotnet/agent-framework-dotnet.slnx | 6 ++ .../responses/Hosted-Steering/.env.example | 5 + .../responses/Hosted-Steering/Dockerfile | 17 ++++ .../Hosted-Steering/Dockerfile.contributor | 18 ++++ .../Hosted-Steering/HostedSteering.csproj | 33 +++++++ .../responses/Hosted-Steering/Program.cs | 68 ++++++++++++++ .../responses/Hosted-Steering/README.md | 74 +++++++++++++++ .../Hosted-Steering/agent.manifest.yaml | 29 ++++++ .../responses/Hosted-Steering/agent.yaml | 9 ++ .../Hosted-Workflow-Resilient/.env.example | 5 + .../Hosted-Workflow-Resilient/Dockerfile | 17 ++++ .../Dockerfile.contributor | 18 ++++ .../HostedWorkflowResilient.csproj | 37 ++++++++ .../Hosted-Workflow-Resilient/Program.cs | 76 +++++++++++++++ .../Hosted-Workflow-Resilient/README.md | 92 +++++++++++++++++++ .../agent.manifest.yaml | 31 +++++++ .../Hosted-Workflow-Resilient/agent.yaml | 9 ++ 17 files changed, 544 insertions(+) create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile.contributor create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.manifest.yaml create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.yaml create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile.contributor create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.manifest.yaml create mode 100644 dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.yaml diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 7bf9266d29c..17c0cae9ac8 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -382,6 +382,12 @@ + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example new file mode 100644 index 00000000000..04335e65b89 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example @@ -0,0 +1,5 @@ +FOUNDRY_PROJECT_ENDPOINT= +ASPNETCORE_URLS=http://+:8088 +ASPNETCORE_ENVIRONMENT=Development +FOUNDRY_MODEL=gpt-4o +AZURE_BEARER_TOKEN=DefaultAzureCredential diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile new file mode 100644 index 00000000000..4f039f42063 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile @@ -0,0 +1,17 @@ +# Use the official .NET 10.0 ASP.NET runtime as a parent image +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src +COPY . . +RUN dotnet restore +RUN dotnet publish -c Release -o /app/publish + +# Final stage +FROM base AS final +WORKDIR /app +COPY --from=build /app/publish . +EXPOSE 8088 +ENV ASPNETCORE_URLS=http://+:8088 +ENTRYPOINT ["dotnet", "HostedSteering.dll"] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile.contributor b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile.contributor new file mode 100644 index 00000000000..7e94ce0cbe3 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Dockerfile.contributor @@ -0,0 +1,18 @@ +# Dockerfile for contributors building from the agent-framework repository source. +# +# This project uses ProjectReference to the local source, which means a standard +# multi-stage Docker build cannot resolve dependencies outside this folder. +# Pre-publish the app targeting the container runtime and copy the output: +# +# dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out +# docker build -f Dockerfile.contributor -t hosted-steering . +# docker run --rm -p 8088:8088 -e AGENT_NAME=hosted-steering -e AZURE_BEARER_TOKEN=$AZURE_BEARER_TOKEN --env-file .env hosted-steering +# +# For end-users consuming the NuGet package (not ProjectReference), use the standard +# Dockerfile which performs a full dotnet restore + publish inside the container. +FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final +WORKDIR /app +COPY out/ . +EXPOSE 8088 +ENV ASPNETCORE_URLS=http://+:8088 +ENTRYPOINT ["dotnet", "HostedSteering.dll"] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj new file mode 100644 index 00000000000..b548d768082 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj @@ -0,0 +1,33 @@ + + + + net10.0 + enable + enable + false + HostedSteering + HostedSteering + $(NoWarn); + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs new file mode 100644 index 00000000000..414845f07b6 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Steerable Conversation Agent — a chat agent hosted as a Foundry Hosted Agent that accepts +// steering: a new input sent while a turn is still running is queued behind the current turn +// instead of being rejected, and the agent picks it up at the next safe point. +// +// What "steering" adds here: +// - The agent is hosted with SteerableConversations = true. A follow-up request for the same +// conversation that is still in progress is accepted (status "queued") rather than rejected +// with a conversation-locked error. +// - Steering is independent of resilience: you can enable either option on its own. This sample +// turns on only steering to keep the behavior focused; see Hosted-Workflow-Resilient for +// crash recovery. +// - Opt-in, off by default. Without SteerableConversations an overlapping turn is rejected, as +// in the non-steering samples. + +using Azure.AI.Projects; +using Azure.Core; +using Azure.Identity; +using DotNetEnv; +using Hosted_Shared_Contributor_Setup; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry.Hosting; + +// Load .env file if present (for local development) +Env.TraversePath().Load(); + +var projectEndpoint = new Uri(Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.")); + +var agentName = Environment.GetEnvironmentVariable("AGENT_NAME") ?? "hosted-steering"; + +var deployment = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-4o"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +TokenCredential credential = new ChainedTokenCredential( + new DevTemporaryTokenCredential(), + new DefaultAzureCredential()); + +// Create the agent via the AI project client using the Responses API. +AIAgent agent = new AIProjectClient(projectEndpoint, credential) + .AsAIAgent( + model: deployment, + instructions: """ + You are a helpful AI assistant hosted as a Foundry Hosted Agent. + When you receive an additional message while already working, treat it as a course + correction and fold it into your ongoing answer. Be concise, clear, and helpful. + """, + name: agentName, + description: "A steerable general-purpose AI assistant"); + +// Host the agent as a Foundry Hosted Agent using the Responses API. +// SteerableConversations opts this host into mid-turn steering; it is the only difference from +// the non-steering Hosted-ChatClientAgent sample. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddFoundryResponses(agent, configure: o => o.SteerableConversations = true); + +var app = builder.Build(); +app.MapFoundryResponses(); + +// Contributor-only: in Development, also map the per-agent OpenAI route shape that live Foundry uses +// so a local REPL client can target this server via AIProjectClient.AsAIAgent(Uri agentEndpoint). +// Do not use this in production. Hosted Foundry agents only support the agent-endpoint path. +app.MapDevTemporaryLocalAgentEndpoint(); + +app.Run(); diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md new file mode 100644 index 00000000000..a0009743b85 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md @@ -0,0 +1,74 @@ +# Hosted-Steering + +A chat agent hosted as a Foundry Hosted Agent using the **Responses protocol**, with **steerable conversations** enabled. A new input sent while a turn is still running is queued behind the current turn and folded into the ongoing answer, instead of being rejected with a conversation-locked error. + +## What "steering" means here + +- **Mid-turn input is queued, not rejected.** With `SteerableConversations = true`, a follow-up request for a conversation that is still in progress is accepted (`status: queued`) and drained at the next safe point, so a user can course-correct without cancelling and restarting. +- **Independent of resilience.** Steering and resilient background responses are separate options; you can enable either on its own. This sample turns on only steering. For crash recovery, see [`Hosted-Workflow-Resilient`](../Hosted-Workflow-Resilient/README.md). +- **Opt-in, off by default.** The only code difference from the non-steering [`Hosted-ChatClientAgent`](../Hosted-ChatClientAgent/README.md) is one line: + + ```csharp + builder.Services.AddFoundryResponses(agent, configure: o => o.SteerableConversations = true); + ``` + + Without the option, an overlapping turn on the same conversation is rejected (`conversation_locked`). + +## Prerequisites + +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) +- `az login` plus a Foundry **project endpoint** and a **model deployment**. + +## Configuration + +```bash +cp .env.example .env +# set FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL +``` + +## Run locally (contributors) + +This project uses `ProjectReference` to build against the local Agent Framework source. + +```bash +az login +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export FOUNDRY_MODEL=gpt-4o + +cd dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering +dotnet run +``` + +The agent starts on `http://localhost:8088`. + +### Try steering + +1. Start a background response for a conversation and note its response id: + + ```bash + curl -N -s http://localhost:8088/responses \ + -H 'content-type: application/json' \ + -d '{"input":"Write a detailed plan for a birthday party","stream":true,"store":true,"background":true}' + ``` + +2. While it is still running, send a follow-up for the same chain (set `previous_response_id` to the latest response id). Instead of `conversation_locked`, it is queued and the agent folds it in: + + ```bash + curl -N -s http://localhost:8088/responses \ + -H 'content-type: application/json' \ + -d '{"input":"Actually, make it a surprise party on a tight budget","previous_response_id":"","stream":true,"store":true,"background":true}' + ``` + +Without `SteerableConversations`, step 2 would be rejected while the first turn is in progress. + +## Deploy to Foundry + +Initialize an `azd` project from this sample's manifest, then deploy: + +```bash +mkdir hosted-steering && cd hosted-steering +azd ai agent init -m https://github.com/microsoft/agent-framework/blob/main/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.manifest.yaml +azd deploy +``` + +See the [official deployment guide](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/deploy-hosted-agent). diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.manifest.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.manifest.yaml new file mode 100644 index 00000000000..6609a469c05 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.manifest.yaml @@ -0,0 +1,29 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/AgentManifest.yaml +name: hosted-steering +displayName: "Steerable Conversation Agent" + +description: > + A chat agent hosted as a Foundry Hosted Agent with steerable conversations enabled, so a new + input sent while a turn is still running is queued behind the current turn and folded into the + ongoing answer instead of being rejected. + +metadata: + tags: + - AI Agent Hosting + - Azure AI AgentServer + - Responses Protocol + - Steering + - Agent Framework + +template: + name: hosted-steering + kind: hosted + protocols: + - protocol: responses + version: 2.0.0 + resources: + cpu: "0.25" + memory: 0.5Gi +parameters: + properties: [] +resources: [] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.yaml new file mode 100644 index 00000000000..e736df727b7 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/agent.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml +kind: hosted +name: hosted-steering +protocols: + - protocol: responses + version: 2.0.0 +resources: + cpu: "0.25" + memory: 0.5Gi diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example new file mode 100644 index 00000000000..04335e65b89 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example @@ -0,0 +1,5 @@ +FOUNDRY_PROJECT_ENDPOINT= +ASPNETCORE_URLS=http://+:8088 +ASPNETCORE_ENVIRONMENT=Development +FOUNDRY_MODEL=gpt-4o +AZURE_BEARER_TOKEN=DefaultAzureCredential diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile new file mode 100644 index 00000000000..2ada9a4d498 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile @@ -0,0 +1,17 @@ +# Use the official .NET 10.0 ASP.NET runtime as a parent image +FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src +COPY . . +RUN dotnet restore +RUN dotnet publish -c Release -o /app/publish + +# Final stage +FROM base AS final +WORKDIR /app +COPY --from=build /app/publish . +EXPOSE 8088 +ENV ASPNETCORE_URLS=http://+:8088 +ENTRYPOINT ["dotnet", "HostedWorkflowResilient.dll"] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile.contributor b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile.contributor new file mode 100644 index 00000000000..087c7c93efc --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Dockerfile.contributor @@ -0,0 +1,18 @@ +# Dockerfile for contributors building from the agent-framework repository source. +# +# This project uses ProjectReference to the local source, which means a standard +# multi-stage Docker build cannot resolve dependencies outside this folder. +# Pre-publish the app targeting the container runtime and copy the output: +# +# dotnet publish -c Debug -f net10.0 -r linux-musl-x64 --self-contained false -o out +# docker build -f Dockerfile.contributor -t hosted-workflow-resilient . +# docker run --rm -p 8088:8088 -e AGENT_NAME=hosted-workflow-resilient -e AZURE_BEARER_TOKEN=$AZURE_BEARER_TOKEN --env-file .env hosted-workflow-resilient +# +# For end-users consuming the NuGet package (not ProjectReference), use the standard +# Dockerfile which performs a full dotnet restore + publish inside the container. +FROM mcr.microsoft.com/dotnet/aspnet:10.0-alpine AS final +WORKDIR /app +COPY out/ . +EXPOSE 8088 +ENV ASPNETCORE_URLS=http://+:8088 +ENTRYPOINT ["dotnet", "HostedWorkflowResilient.dll"] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj new file mode 100644 index 00000000000..1753aa29e26 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj @@ -0,0 +1,37 @@ + + + + net10.0 + enable + enable + false + HostedWorkflowResilient + HostedWorkflowResilient + $(NoWarn); + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs new file mode 100644 index 00000000000..aec5e0efab1 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Resilient Translation Chain Workflow Agent — the same sequential translation workflow as +// Hosted-Workflow-Simple, hosted as a durable long-running (resilient) Foundry Hosted Agent. +// +// What "resilient" adds here: +// - The workflow is hosted with ResilientBackground = true. For a background response +// (store=true, background=true), the platform keeps the agent running with no client +// connected and re-invokes the handler after a container crash or graceful shutdown. +// - A workflow hosted as an agent checkpoints its progress between executor steps and resumes +// from its last checkpoint. The hosting handler persists the session at each completed output +// item, so a crash mid-run loses at most the step that was in flight. +// - Everything is opt-in: without ResilientBackground the agent behaves exactly like the +// non-resilient sample. +// +// See the README for the local crash-and-recover walkthrough. + +using Azure.AI.Projects; +using Azure.Core; +using Azure.Identity; +using DotNetEnv; +using Hosted_Shared_Contributor_Setup; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry.Hosting; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +// Load .env file if present (for local development) +Env.TraversePath().Load(); + +string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +string deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-4o"; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +TokenCredential credential = new ChainedTokenCredential( + new DevTemporaryTokenCredential(), + new DefaultAzureCredential()); + +// Create a chat client from the Foundry project +IChatClient chatClient = new AIProjectClient(new Uri(endpoint), credential) + .GetProjectOpenAIClient() + .GetChatClient(deploymentName) + .AsIChatClient(); + +// Create translation agents. Each becomes a workflow step, so each completed translation is a +// natural checkpoint boundary the platform can resume from. +AIAgent frenchAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to French."); +AIAgent spanishAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to Spanish."); +AIAgent englishAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to English."); + +// Build the sequential workflow: French → Spanish → English +AIAgent agent = new WorkflowBuilder(frenchAgent) + .AddEdge(frenchAgent, spanishAgent) + .AddEdge(spanishAgent, englishAgent) + .Build() + .AsAIAgent( + name: Environment.GetEnvironmentVariable("AGENT_NAME") ?? "hosted-workflow-resilient"); + +// Host the workflow agent as a durable Foundry Hosted Agent using the Responses API. +// ResilientBackground opts this host into crash-recoverable background responses; it is the only +// difference from the non-resilient Hosted-Workflow-Simple sample. +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddFoundryResponses(agent, configure: o => o.ResilientBackground = true); + +var app = builder.Build(); +app.MapFoundryResponses(); + +// Contributor-only: in Development, also map the per-agent OpenAI route shape that live Foundry uses +// so a local REPL client can target this server via AIProjectClient.AsAIAgent(Uri agentEndpoint). +// Do not use this in production. Hosted Foundry agents only support the agent-endpoint path. +app.MapDevTemporaryLocalAgentEndpoint(); + +app.Run(); diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md new file mode 100644 index 00000000000..c8a4144c508 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md @@ -0,0 +1,92 @@ +# Hosted-Workflow-Resilient + +A durable, long-running **workflow** hosted as a Foundry Hosted Agent using the **Responses protocol**. It is the same English to French to Spanish back to English translation chain as [`Hosted-Workflow-Simple`](../Hosted-Workflow-Simple/README.md), with one difference: it opts into **resilient background responses**, so a background response survives a container crash or graceful shutdown and resumes from the workflow's last completed step. + +## What "resilient" means here + +- **Long-running with no client connected.** When a caller starts a background response (`store: true`, `background: true`), the platform keeps the agent running even if the caller disconnects. +- **Crash recovery.** If the container crashes or is recycled mid-run, the platform restarts it and re-invokes the handler. A workflow hosted as an agent checkpoints its progress between steps, so it resumes from its last completed step instead of restarting from scratch. +- **At most one step repeats.** The hosting handler persists the session at each completed output item (a natural workflow step boundary), so a crash loses at most the step that was in flight. +- **Opt-in, off by default.** The only code difference from the non-resilient sample is one line: + + ```csharp + builder.Services.AddFoundryResponses(agent, configure: o => o.ResilientBackground = true); + ``` + + Durability applies only to background responses. A foreground response (the caller waits on the connection) is not durable: a crash simply fails it. + +## Prerequisites + +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) +- `az login` plus a Foundry **project endpoint** and a **model deployment** (each translation step calls the model). + +## Configuration + +```bash +cp .env.example .env +# set FOUNDRY_PROJECT_ENDPOINT and FOUNDRY_MODEL +``` + +## Run locally (contributors) + +This project uses `ProjectReference` to build against the local Agent Framework source. + +```bash +az login +export FOUNDRY_PROJECT_ENDPOINT=https://.services.ai.azure.com/api/projects/ +export FOUNDRY_MODEL=gpt-4o + +cd dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient +dotnet run +``` + +The agent starts on `http://localhost:8088`. + +### Local crash-and-recover walkthrough + +Resilient recovery needs a state store that survives a process restart. Locally the SDK auto-selects a file-backed store when `FOUNDRY_HOSTING_ENVIRONMENT` is unset; pin the store root and the session id so a restart finds the in-progress response: + +```bash +export AGENTSERVER_STATE_ROOT=$PWD/.agentserver-state +export FOUNDRY_AGENT_SESSION_ID=local-demo-session +dotnet run +``` + +1. Start a background response and stream it. Capture the response id (`"id":"caresp_..."`): + + ```bash + curl -N -s http://localhost:8088/responses \ + -H 'content-type: application/json' \ + -d '{"input":"renewable energy supply chains","stream":true,"store":true,"background":true}' + ``` + +2. After a translation step or two, stop the process (Ctrl+C, or kill it) to simulate a crash. + +3. Restart against the **same** `AGENTSERVER_STATE_ROOT` and `FOUNDRY_AGENT_SESSION_ID`. On startup the resilient task scanner reclaims the in-progress response and re-invokes the handler, which resumes the workflow from its last completed step. + +4. Reconnect and watch it finish: + + ```bash + curl -N -s "http://localhost:8088/responses/?stream=true" + ``` + +## How local mode works + +| Env var | Effect | +|---|---| +| `FOUNDRY_HOSTING_ENVIRONMENT` (**unset**) | The SDK auto-selects the file-backed store instead of the hosted task API. | +| `AGENTSERVER_STATE_ROOT` | Where the resilient task store lives (must survive the restart). | +| `FOUNDRY_AGENT_SESSION_ID` | The session pinned across restarts so recovery finds the in-progress response. | +| `HOME` | The session working directory. The agent session store writes under `{HOME}/.checkpoints`; in a hosted container this is a durable per-session volume. | + +## Deploy to Foundry + +Initialize an `azd` project from this sample's manifest, then deploy: + +```bash +mkdir hosted-workflow-resilient && cd hosted-workflow-resilient +azd ai agent init -m https://github.com/microsoft/agent-framework/blob/main/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.manifest.yaml +azd deploy +``` + +Drive it with a background response (`"background": true`), then exercise crash recovery by letting the platform restart the container. See the [official deployment guide](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/deploy-hosted-agent). diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.manifest.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.manifest.yaml new file mode 100644 index 00000000000..146a85e45c7 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.manifest.yaml @@ -0,0 +1,31 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/AgentManifest.yaml +name: hosted-workflow-resilient +displayName: "Resilient Translation Chain Workflow Agent" + +description: > + A durable long-running workflow agent that performs sequential translation through multiple + languages (English to French to Spanish back to English). It is hosted with resilient background + responses enabled, so a background response survives a container crash or graceful shutdown and + resumes from the workflow's last completed step. + +metadata: + tags: + - AI Agent Hosting + - Azure AI AgentServer + - Responses Protocol + - Workflows + - Resilient + - Agent Framework + +template: + name: hosted-workflow-resilient + kind: hosted + protocols: + - protocol: responses + version: 2.0.0 + resources: + cpu: "0.25" + memory: 0.5Gi +parameters: + properties: [] +resources: [] diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.yaml new file mode 100644 index 00000000000..2afd7099157 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/agent.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/microsoft/AgentSchema/refs/heads/main/schemas/v1.0/ContainerAgent.yaml +kind: hosted +name: hosted-workflow-resilient +protocols: + - protocol: responses + version: 2.0.0 +resources: + cpu: "0.25" + memory: 0.5Gi From bf3ccfa7336c0c278f11f7b4a2792da8b9214b17 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:15:18 +0100 Subject: [PATCH 05/18] Close the workflow crash-recovery re-entry in the hosting handler On a crash-recovery re-invocation (context.IsRecovery), align the handler with the SDK's resilient pattern so a resumed background response reaches a terminal state instead of stalling: - Seed the event stream from the last durable snapshot (context.PersistedResponse) instead of the request, so the resumed stream continues from the output items already emitted before the crash. The output-index allocator advances to PersistedResponse.Output.Count and re-emitted items do not collide with the already-durable slots. - Skip re-injecting the original input on recovery. The persisted session already carries the progress (a workflow hosted as an agent resumes from its last checkpoint, restored by GetSessionAsync), so re-injecting would enqueue a duplicate turn on top of the resumed state and the run would never terminate. Both changes are gated on IsRecovery, so a normal (non-recovery) turn is unchanged. --- .../AgentFrameworkResponseHandler.cs | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index d31182d8966..7e428ec81a5 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -178,8 +178,16 @@ public override async IAsyncEnumerable CreateAsync( } } - // 3. Create the SDK event stream builder - var stream = new ResponseEventStream(context, request); + // 3. Create the SDK event stream builder. + // On a crash-recovery re-invocation, seed the stream from the last durable snapshot + // (context.PersistedResponse) instead of the request, so the resumed stream continues from + // the output items already emitted before the crash: the output-index allocator advances to + // PersistedResponse.Output.Count and re-emitted items do not collide with the already-durable + // slots. On a fresh invocation, seed from the request as before. This is gated on IsRecovery, + // so a non-resilient turn is unchanged. + var stream = context.IsRecovery && context.PersistedResponse is { } persistedResponse + ? new ResponseEventStream(context, persistedResponse) + : new ResponseEventStream(context, request); // 3. Emit lifecycle events yield return stream.EmitCreated(); @@ -188,31 +196,40 @@ public override async IAsyncEnumerable CreateAsync( // 4. Convert input: history + current input → ChatMessage[] var messages = new List(); - // Load conversation history only for fresh sessions. When a session already exists - // (e.g. resuming a workflow paused at an external-input port), the workflow's - // checkpointed state already contains the prior turns' messages — replaying history - // would re-drive completed actions and break HITL resume semantics. - var isResume = (!string.IsNullOrWhiteSpace(conversationId) || !string.IsNullOrWhiteSpace(request.PreviousResponseId)) - && session?.StateBag?.Count > 0; - if (!isResume) + // On a crash-recovery re-invocation the platform re-delivers the same input, but the + // persisted session already carries the progress: a workflow hosted as an agent resumes from + // its last checkpoint (restored above by GetSessionAsync). Re-injecting the original input + // would enqueue a duplicate turn on top of the resumed state, so the run never reaches a + // terminal state. Leave messages empty on recovery and let the restored session drive the + // resume to completion. + if (!context.IsRecovery) { - var history = await context.GetHistoryAsync(cancellationToken).ConfigureAwait(false); - if (history.Count > 0) + // Load conversation history only for fresh sessions. When a session already exists + // (e.g. resuming a workflow paused at an external-input port), the workflow's + // checkpointed state already contains the prior turns' messages — replaying history + // would re-drive completed actions and break HITL resume semantics. + var isResume = (!string.IsNullOrWhiteSpace(conversationId) || !string.IsNullOrWhiteSpace(request.PreviousResponseId)) + && session?.StateBag?.Count > 0; + if (!isResume) { - messages.AddRange(InputConverter.ConvertOutputItemsToMessages(history, session?.StateBag)); + var history = await context.GetHistoryAsync(cancellationToken).ConfigureAwait(false); + if (history.Count > 0) + { + messages.AddRange(InputConverter.ConvertOutputItemsToMessages(history, session?.StateBag)); + } } - } - // Load and convert current input items - var inputItems = await context.GetInputItemsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - if (inputItems.Count > 0) - { - messages.AddRange(InputConverter.ConvertItemsToMessages(inputItems, session?.StateBag)); - } - else - { - // Fall back to raw request input - messages.AddRange(InputConverter.ConvertInputToMessages(request, session?.StateBag)); + // Load and convert current input items + var inputItems = await context.GetInputItemsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + if (inputItems.Count > 0) + { + messages.AddRange(InputConverter.ConvertItemsToMessages(inputItems, session?.StateBag)); + } + else + { + // Fall back to raw request input + messages.AddRange(InputConverter.ConvertInputToMessages(request, session?.StateBag)); + } } // 5. Build chat options From 8a9b05a09180be1210fd4d0982c1bcbe5f9809d6 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:15:31 +0100 Subject: [PATCH 06/18] Give the resilient workflow sample stable executor ids A workflow checkpoint records each step by its executor id, and an agent-backed step derives that id from the agent's Id (and Name). By default an agent gets a fresh random Id per process, so after a crash the restarted process would rebuild the workflow with different ids and the saved checkpoint would no longer match, failing the resume with "checkpoint is not compatible with the workflow". Give each translation agent a fixed Id and Name so the rebuilt workflow is identical across restarts and the checkpoint resumes cleanly. A code comment spells out why this matters for resilient workflows. --- .../Hosted-Workflow-Resilient/Program.cs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs index aec5e0efab1..ff12be41b4f 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs @@ -47,9 +47,30 @@ // Create translation agents. Each becomes a workflow step, so each completed translation is a // natural checkpoint boundary the platform can resume from. -AIAgent frenchAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to French."); -AIAgent spanishAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to Spanish."); -AIAgent englishAgent = chatClient.AsAIAgent("You are a translation assistant that translates the provided text to English."); +// +// IMPORTANT for resilient workflows: give every agent a STABLE Id. A workflow checkpoint records +// each step by its executor id, and an agent-backed step derives that id from the agent's Id (and +// Name). By default an agent gets a fresh random Id per process, so after a crash the restarted +// process would rebuild the workflow with different ids and the saved checkpoint would no longer +// match, failing the resume. Fixed ids keep the rebuilt workflow identical across restarts. +AIAgent frenchAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "french-translator", + Name = "french-translator", + ChatOptions = new() { Instructions = "You are a translation assistant that translates the provided text to French." }, +}); +AIAgent spanishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "spanish-translator", + Name = "spanish-translator", + ChatOptions = new() { Instructions = "You are a translation assistant that translates the provided text to Spanish." }, +}); +AIAgent englishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "english-translator", + Name = "english-translator", + ChatOptions = new() { Instructions = "You are a translation assistant that translates the provided text to English." }, +}); // Build the sequential workflow: French → Spanish → English AIAgent agent = new WorkflowBuilder(frenchAgent) From b1a7601e14a06b26fe5c626f84304184a32bd573 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:39:55 +0100 Subject: [PATCH 07/18] Expose signals to detect unstable workflow executor ids Add the minimal, non-public hooks the Foundry resilient-hosting readiness check needs to detect, ahead of time, a workflow whose agent-backed steps would not survive a crash-recovery resume: - AIAgent.HasAutoGeneratedId (internal): true when no explicit id was supplied (IdCore is null), so Id falls back to a per-process random value. This is the authoritative signal: a caller-supplied id that happens to be a GUID is correctly reported as stable. Exposed to the hosting package via a new InternalsVisibleTo entry on the abstractions assembly. - WorkflowHostAgent.GetService now returns the hosted Workflow for an unkeyed Workflow service request, so a host can inspect the workflow's executor ids (and their backing agents) without constructing a session. Additive, with no change to existing behaviour. WorkflowHostAgentTests covers the GetService enabler. --- .../AIAgent.cs | 12 +++++++ .../Microsoft.Agents.AI.Abstractions.csproj | 1 + .../WorkflowHostAgent.cs | 21 ++++++++++++ .../WorkflowHostAgentTests.cs | 33 +++++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostAgentTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgent.cs b/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgent.cs index 3431a4b52b1..18a361f2de4 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/AIAgent.cs @@ -69,6 +69,18 @@ public abstract partial class AIAgent /// protected virtual string? IdCore => null; + /// + /// Gets a value indicating whether is an auto-generated identifier, that is, no + /// explicit id was supplied by the caller or backing service ( is ), + /// so falls back to a value generated fresh per process. + /// + /// + /// Hosting uses this to detect, ahead of time, agents whose ids change on every process start. + /// Such ids break crash-recovery resume of a workflow hosted as an agent, which matches its + /// checkpoint to the rebuilt workflow by executor id. + /// + internal bool HasAutoGeneratedId => this.IdCore is null; + /// /// Gets the human-readable name of the agent. /// diff --git a/dotnet/src/Microsoft.Agents.AI.Abstractions/Microsoft.Agents.AI.Abstractions.csproj b/dotnet/src/Microsoft.Agents.AI.Abstractions/Microsoft.Agents.AI.Abstractions.csproj index 9f5668c812a..ad7ea577313 100644 --- a/dotnet/src/Microsoft.Agents.AI.Abstractions/Microsoft.Agents.AI.Abstractions.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Abstractions/Microsoft.Agents.AI.Abstractions.csproj @@ -32,5 +32,6 @@ + diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowHostAgent.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowHostAgent.cs index 01f96379674..e5b37d47aae 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowHostAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowHostAgent.cs @@ -55,6 +55,27 @@ public WorkflowHostAgent(Workflow workflow, string? id = null, string? name = nu public override string? Name { get; } public override string? Description { get; } + /// + /// + /// In addition to the base behaviour, this returns the hosted when + /// requested with an unkeyed service type. This lets a host inspect the + /// workflow (for example, its executor ids via and + /// ) without constructing a session. It is used by the + /// Foundry resilient-hosting readiness check to detect auto-generated executor ids that would + /// break crash-recovery resume. + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + if (serviceKey is null && serviceType == typeof(Workflow)) + { + return this._workflow; + } + + return base.GetService(serviceType, serviceKey); + } + private string GenerateNewId() { string result; diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostAgentTests.cs new file mode 100644 index 00000000000..bc0e1892329 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostAgentTests.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using FluentAssertions; + +namespace Microsoft.Agents.AI.Workflows.UnitTests; + +public class WorkflowHostAgentTests +{ + [Fact] + public void GetService_Workflow_ReturnsHostedWorkflow() + { + // Arrange + var french = new TestEchoAgent("french", "french"); + var spanish = new TestEchoAgent("spanish", "spanish"); + var workflow = new WorkflowBuilder(french).AddEdge(french, spanish).Build(); + + // Act + var host = workflow.AsAIAgent(name: "host"); + + // Assert: the host exposes the inner workflow so callers can inspect its executor ids. + host.GetService().Should().BeSameAs(workflow); + } + + [Fact] + public void GetService_Workflow_OnPlainAgent_ReturnsNull() + { + // Arrange + var agent = new TestEchoAgent("solo", "solo"); + + // Act + Assert: a non-workflow agent does not resolve a Workflow service. + agent.GetService().Should().BeNull(); + } +} From 977d44e6b528dfb7a7159fb802a7bf6f1bfc25ec Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:40:16 +0100 Subject: [PATCH 08/18] Fail readiness for resilient workflows with unstable executor ids A workflow hosted as an agent resumes after a crash by matching its checkpoint to the rebuilt workflow by executor id. An agent-backed step derives that id from the agent's id, so an agent with an auto-generated id produces a different id on every start and the resume fails deep in the engine with "the specified checkpoint is not compatible with the workflow". Add a /readiness health check that, only while ResilientBackground is on, inspects every registered workflow-host agent (via GetService) and reports Unhealthy with an actionable message when any agent-backed step has an auto-generated id (AIAgent.HasAutoGeneratedId). This turns a late, cryptic resume failure into an early not-ready signal. It reads already-constructed agent instances from the binding raw value, so a caller-supplied GUID id is correctly treated as stable, and it never creates a session. On a non-resilient host it is a no-op. Because the check reads abstractions internals via InternalsVisibleTo, the hosting assembly now consumes the shared Throw and DiagnosticIds sources from the abstractions assembly instead of injecting its own copies, matching how other projects consume a referenced assembly's shared sources. --- ...Microsoft.Agents.AI.Foundry.Hosting.csproj | 2 - .../ResilientWorkflowExecutorIdHealthCheck.cs | 92 +++++++++++ .../ServiceCollectionExtensions.cs | 59 +++++++ ...lientWorkflowExecutorIdHealthCheckTests.cs | 155 ++++++++++++++++++ 4 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientWorkflowExecutorIdHealthCheckTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/Microsoft.Agents.AI.Foundry.Hosting.csproj b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/Microsoft.Agents.AI.Foundry.Hosting.csproj index e1d1cf5a672..80de75611e2 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/Microsoft.Agents.AI.Foundry.Hosting.csproj +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/Microsoft.Agents.AI.Foundry.Hosting.csproj @@ -10,8 +10,6 @@ - true - true true true $(NoWarn);OPENAI001;MEAI001;MAAI001;NU1903 diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs new file mode 100644 index 00000000000..f928c0294ac --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI.Foundry.Hosting; + +/// +/// Readiness health check that, on a resilient host, reports unhealthy when a workflow hosted as an +/// agent has one or more steps whose agent has an auto-generated id. +/// +/// +/// +/// Crash-recovery resume matches the saved checkpoint to the rebuilt workflow by executor id (see +/// WorkflowInfo.IsMatch). An agent-backed executor derives its id from the agent's +/// , which, when the caller supplies no explicit id, defaults to a fresh +/// random value per process. A workflow built from such agents produces different ids on every +/// start, so after a container restart the resumed run fails deep in the engine with "the specified +/// checkpoint is not compatible with the workflow". +/// +/// +/// This check turns that late, cryptic failure into an early, actionable not-ready signal: while +/// is on, it +/// inspects every registered workflow-host agent (via agent.GetService<Workflow>()) and +/// fails readiness if any agent-backed step has an auto-generated id +/// (). This is authoritative: a caller-supplied id that +/// happens to be a GUID is correctly treated as stable. On a non-resilient host it is a no-op +/// (always healthy), because unstable ids are harmless when there is no recovery. It has no side +/// effects: it reads already-constructed agent instances and never creates a session. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +internal sealed class ResilientWorkflowExecutorIdHealthCheck : IHealthCheck +{ + private readonly Func _isResilient; + private readonly Func> _agents; + + public ResilientWorkflowExecutorIdHealthCheck(Func isResilient, Func> agents) + { + ArgumentNullException.ThrowIfNull(isResilient); + ArgumentNullException.ThrowIfNull(agents); + this._isResilient = isResilient; + this._agents = agents; + } + + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + // Unstable executor ids only matter when the host can recover a run after a restart. + if (!this._isResilient()) + { + return Task.FromResult(HealthCheckResult.Healthy( + description: "Resilient workflow ids: not applicable (ResilientBackground is off).")); + } + + List? offenders = null; + foreach (var agent in this._agents()) + { + if (agent.GetService() is not { } workflow) + { + // Not a workflow-host agent; single agents have no internal executor ids to match. + continue; + } + + foreach (var (executorId, binding) in workflow.ReflectExecutors()) + { + // An agent-backed executor exposes its agent as the binding's raw value. Only those + // derive an id from an AIAgent, so only those can carry an auto-generated id. + if (binding.RawValue is AIAgent stepAgent && stepAgent.HasAutoGeneratedId) + { + (offenders ??= []).Add($"agent '{agent.Name ?? agent.Id}' step '{executorId}'"); + } + } + } + + if (offenders is null) + { + return Task.FromResult(HealthCheckResult.Healthy( + description: "Resilient workflow ids: all hosted workflow steps have stable ids.")); + } + + return Task.FromResult(new HealthCheckResult( + status: context.Registration.FailureStatus, + description: + $"Resilient workflow ids: one or more hosted workflow steps use an agent with an auto-generated id, which breaks crash-recovery resume (the checkpoint is matched to the rebuilt workflow by executor id, and an auto-generated id changes on every restart). Assign a stable Id to each agent in the workflow, for example chatClient.AsAIAgent(options: new() {{ Id = \"translator\", Name = \"translator\" }}). Offending steps: {string.Join("; ", offenders)}.")); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs index fbd1ba3b00e..94282bcb466 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs @@ -2,6 +2,7 @@ using System; using System.ClientModel.Primitives; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; @@ -322,6 +323,7 @@ private static void AddFoundryResponsesServer(IServiceCollection services, Actio services.AddResponsesServer(configure ?? (_ => { })); AddAgentRegistrationHealthCheck(services); + AddResilientWorkflowIdHealthCheck(services); } /// @@ -354,6 +356,63 @@ private static void AddAgentRegistrationHealthCheck(IServiceCollection services) }); } + /// + /// Registers a /readiness health check that, while + /// is on, reports unhealthy when a + /// workflow hosted as an agent has auto-generated executor ids. Such ids change every process, + /// so a crash-recovery resume would fail to match its checkpoint. Surfacing this at readiness + /// turns a late, cryptic resume failure into an early, actionable not-ready signal. On a + /// non-resilient host the check is a no-op. Dedupes by name so a repeat registration is ignored. + /// + private static void AddResilientWorkflowIdHealthCheck(IServiceCollection services) + { + const string HealthCheckName = "foundry-resilient-workflow-ids"; + services.AddHealthChecks(); + services.Configure(opts => + { + foreach (var existing in opts.Registrations) + { + if (string.Equals(existing.Name, HealthCheckName, StringComparison.Ordinal)) + { + return; + } + } + + opts.Registrations.Add(new HealthCheckRegistration( + name: HealthCheckName, + factory: sp => new ResilientWorkflowExecutorIdHealthCheck( + isResilient: () => sp.GetService>()?.Value.ResilientBackground ?? false, + agents: () => EnumerateRegisteredAgents(services)), + failureStatus: HealthStatus.Unhealthy, + tags: ["foundry", "workflow", "readiness"])); + }); + } + + /// + /// Enumerates the instances registered on as + /// concrete singletons (default or keyed). Reads the descriptors directly so no agent is + /// resolved or constructed; agents registered via a factory (rather than an instance) are not + /// inspectable this way and are skipped. + /// + private static IEnumerable EnumerateRegisteredAgents(IServiceCollection services) + { + foreach (var descriptor in services) + { + if (descriptor.ServiceType != typeof(AIAgent)) + { + continue; + } + + object? instance = descriptor.IsKeyedService + ? descriptor.KeyedImplementationInstance + : descriptor.ImplementationInstance; + if (instance is AIAgent agent) + { + yield return agent; + } + } + } + /// /// The ActivitySource name for the Responses hosting pipeline. /// diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientWorkflowExecutorIdHealthCheckTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientWorkflowExecutorIdHealthCheckTests.cs new file mode 100644 index 00000000000..24aca8f0781 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientWorkflowExecutorIdHealthCheckTests.cs @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Moq; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +public class ResilientWorkflowExecutorIdHealthCheckTests +{ + [Fact] + public async Task CheckHealthAsync_NotResilient_ReturnsHealthyEvenWithAutoIdsAsync() + { + // Arrange: a workflow with auto-generated executor ids, but the host is not resilient. + var host = BuildWorkflowAgent(stableIds: false); + var check = new ResilientWorkflowExecutorIdHealthCheck(isResilient: () => false, agents: () => [host]); + + // Act + var result = await check.CheckHealthAsync(NewContext()); + + // Assert: unstable ids are harmless without recovery, so the check is a no-op. + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + [Fact] + public async Task CheckHealthAsync_Resilient_StableIds_ReturnsHealthyAsync() + { + // Arrange: resilient host, workflow whose agents have explicit stable ids. + var host = BuildWorkflowAgent(stableIds: true); + var check = new ResilientWorkflowExecutorIdHealthCheck(isResilient: () => true, agents: () => [host]); + + // Act + var result = await check.CheckHealthAsync(NewContext()); + + // Assert + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + [Fact] + public async Task CheckHealthAsync_Resilient_AutoGeneratedIds_ReturnsUnhealthyAsync() + { + // Arrange: resilient host, workflow whose agents rely on the default random id. + var host = BuildWorkflowAgent(stableIds: false); + var check = new ResilientWorkflowExecutorIdHealthCheck(isResilient: () => true, agents: () => [host]); + + // Act + var result = await check.CheckHealthAsync(NewContext()); + + // Assert: the auto-generated ids break resume, so readiness fails with an actionable message. + Assert.Equal(HealthStatus.Unhealthy, result.Status); + Assert.Contains("stable Id", result.Description, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public async Task CheckHealthAsync_Resilient_ExplicitGuidId_ReturnsHealthyAsync() + { + // Arrange: resilient host whose agents were given explicit ids that happen to be GUIDs. + // This is the case the previous string heuristic got wrong: a caller-supplied GUID is a + // stable id, so it must be treated as healthy. HasAutoGeneratedId is false here. + var a = new IdAgent(Guid.NewGuid().ToString("N"), name: "french"); + var b = new IdAgent(Guid.NewGuid().ToString("N"), name: "spanish"); + var host = new WorkflowBuilder(a).AddEdge(a, b).Build().AsAIAgent(name: "host"); + var check = new ResilientWorkflowExecutorIdHealthCheck(isResilient: () => true, agents: () => [host]); + + // Act + var result = await check.CheckHealthAsync(NewContext()); + + // Assert + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + [Fact] + public async Task CheckHealthAsync_Resilient_SingleAgentWithoutWorkflow_ReturnsHealthyAsync() + { + // Arrange: a plain (non-workflow) agent exposes no internal executor ids to match. + var single = new IdAgent(id: null, name: "single"); + var check = new ResilientWorkflowExecutorIdHealthCheck(isResilient: () => true, agents: () => [single]); + + // Act + var result = await check.CheckHealthAsync(NewContext()); + + // Assert + Assert.Equal(HealthStatus.Healthy, result.Status); + } + + private static AIAgent BuildWorkflowAgent(bool stableIds) + { + var a = stableIds ? new IdAgent("french", "french") : new IdAgent(id: null, name: null); + var b = stableIds ? new IdAgent("spanish", "spanish") : new IdAgent(id: null, name: null); + return new WorkflowBuilder(a).AddEdge(a, b).Build().AsAIAgent(name: "host"); + } + + private static HealthCheckContext NewContext() => + new() + { + Registration = new HealthCheckRegistration( + name: "foundry-resilient-workflow-ids", + instance: Mock.Of(), + failureStatus: HealthStatus.Unhealthy, + tags: null), + }; + + /// + /// Minimal agent whose id is stable when an explicit id is supplied, and defaults to the + /// framework's random per-process id when it is not. Used to build workflows with either stable + /// or auto-generated executor ids. + /// + private sealed class IdAgent : AIAgent + { + private readonly string? _id; + + public IdAgent(string? id, string? name) + { + this._id = id; + this.Name = name; + } + + protected override string? IdCore => this._id; + public override string? Name { get; } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + System.Text.Json.JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask DeserializeSessionCoreAsync( + System.Text.Json.JsonElement serializedState, + System.Text.Json.JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + } +} From e761b4efede2b4366984e39cecfaa9504c8611c6 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:40:32 +0100 Subject: [PATCH 09/18] Refine stable-id guidance in the resilient workflow sample Use options: new() to set each agent's stable Id and Name (the named argument picks the options overload so the target-typed new() is unambiguous), and add a README note explaining why resilient workflows need stable executor ids. --- .../responses/Hosted-Workflow-Resilient/Program.cs | 6 +++--- .../responses/Hosted-Workflow-Resilient/README.md | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs index ff12be41b4f..893ce980df7 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs @@ -53,19 +53,19 @@ // Name). By default an agent gets a fresh random Id per process, so after a crash the restarted // process would rebuild the workflow with different ids and the saved checkpoint would no longer // match, failing the resume. Fixed ids keep the rebuilt workflow identical across restarts. -AIAgent frenchAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +AIAgent frenchAgent = chatClient.AsAIAgent(options: new() { Id = "french-translator", Name = "french-translator", ChatOptions = new() { Instructions = "You are a translation assistant that translates the provided text to French." }, }); -AIAgent spanishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +AIAgent spanishAgent = chatClient.AsAIAgent(options: new() { Id = "spanish-translator", Name = "spanish-translator", ChatOptions = new() { Instructions = "You are a translation assistant that translates the provided text to Spanish." }, }); -AIAgent englishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +AIAgent englishAgent = chatClient.AsAIAgent(options: new() { Id = "english-translator", Name = "english-translator", diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md index c8a4144c508..10941013bcf 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md @@ -7,7 +7,18 @@ A durable, long-running **workflow** hosted as a Foundry Hosted Agent using the - **Long-running with no client connected.** When a caller starts a background response (`store: true`, `background: true`), the platform keeps the agent running even if the caller disconnects. - **Crash recovery.** If the container crashes or is recycled mid-run, the platform restarts it and re-invokes the handler. A workflow hosted as an agent checkpoints its progress between steps, so it resumes from its last completed step instead of restarting from scratch. - **At most one step repeats.** The hosting handler persists the session at each completed output item (a natural workflow step boundary), so a crash loses at most the step that was in flight. -- **Opt-in, off by default.** The only code difference from the non-resilient sample is one line: +- **Stable executor ids.** Recovery matches the saved checkpoint to the rebuilt workflow by executor id, and an agent-backed step derives its id from the agent's id. A default agent gets a fresh random id per process, which would never match after a restart, so each agent is created with an explicit stable `Id`: + + ```csharp + AIAgent frenchAgent = chatClient.AsAIAgent(options: new() + { + Id = "french-translator", + Name = "french-translator", + ChatOptions = new() { Instructions = "...translate to French." }, + }); + ``` + +- **Opt-in, off by default.** Turning on resilience is one line: ```csharp builder.Services.AddFoundryResponses(agent, configure: o => o.ResilientBackground = true); From a86ec5236c588b5224255fad74ea58ee04726734 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:40:32 +0100 Subject: [PATCH 10/18] Document SUT-centric test file naming convention Record in the .NET AGENTS.md that a test file is named after its system under test (the type tested), and that a single file should not mix multiple systems under test. --- dotnet/AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/AGENTS.md b/dotnet/AGENTS.md index 7cf79f31201..bf079cb01a5 100644 --- a/dotnet/AGENTS.md +++ b/dotnet/AGENTS.md @@ -40,7 +40,7 @@ using types like `IChatClient`, `FunctionInvokingChatClient`, `AITool`, `AIFunct - **Async**: Use `Async` suffix for methods returning `Task`/`ValueTask` - **Private classes**: Should be `sealed` unless subclassed - **Config**: Read from environment variables with `UPPER_SNAKE_CASE` naming -- **Tests**: Add Arrange/Act/Assert comments; use Moq for mocking; test methods returning `Task`/`ValueTask` must use the `Async` suffix. +- **Tests**: Add Arrange/Act/Assert comments; use Moq for mocking; test methods returning `Task`/`ValueTask` must use the `Async` suffix. Name a test file after its system under test (the type being tested), not after the behaviour or feature under test: for example, tests for `WorkflowHostAgent` live in `WorkflowHostAgentTests.cs`. Keep tests for a given type in that type's test file rather than mixing multiple systems under test in one file. ## Key Design Principles From a95a44d8de8843c8f9fc25de85fb328235088aae Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:08:12 +0100 Subject: [PATCH 11/18] Detect a resume by a specific marker, not by state-bag count The non-recovery history-replay guard treated a session as a resume when its state bag held any entry. A hosted session is stamped once, on its first turn, with a write-once HostedSessionContext, which makes the state bag non-empty from the first turn, so counting entries could make a brand new conversation look like a resume and skip loading the history of a conversation the platform resumed by id. Decide instead from a specific marker captured at the right time: whether the session already carried a HostedSessionContext before this turn stamped one. A session the store loaded from a prior turn has it; a freshly created session does not. This reflects whether a prior turn established the session without relying on how many state-bag entries happen to exist. In local (non-hosted) use no context is stamped, so a fresh turn loads history as before. --- .../AgentFrameworkResponseHandler.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index 7e428ec81a5..676f69f167e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -155,6 +155,13 @@ public override async IAsyncEnumerable CreateAsync( var platformCallId = context.PlatformContext?.CallId; HostedCallContext.CallId = platformCallId; + // Capture, before the stamping block below writes it, whether this session was already + // established by a prior turn. On a hosted turn the store loads a session that a prior turn + // saved together with its write-once HostedSessionContext; a freshly created session has none + // yet. This specific marker (not a count of state-bag entries) tells a resume from a first + // turn. + var sessionEstablishedByPriorTurn = session?.GetHostedContext() is not null; + // Stamp/validate the hosted identity only when one was resolved. Locally (non-hosted) there is // no user identity, so there is nothing to partition or tamper-check and the session is shared. if (session is not null && resolvedHostedContext is not null) @@ -204,12 +211,13 @@ public override async IAsyncEnumerable CreateAsync( // resume to completion. if (!context.IsRecovery) { - // Load conversation history only for fresh sessions. When a session already exists - // (e.g. resuming a workflow paused at an external-input port), the workflow's - // checkpointed state already contains the prior turns' messages — replaying history - // would re-drive completed actions and break HITL resume semantics. + // Load conversation history only for fresh sessions. When a session was already + // established by a prior turn (e.g. resuming a workflow paused at an external-input port), + // its checkpointed state already contains those messages — replaying history would + // re-drive completed actions and break HITL resume semantics. The signal is the specific + // HostedSessionContext marker captured above, not a count of state-bag entries. var isResume = (!string.IsNullOrWhiteSpace(conversationId) || !string.IsNullOrWhiteSpace(request.PreviousResponseId)) - && session?.StateBag?.Count > 0; + && sessionEstablishedByPriorTurn; if (!isResume) { var history = await context.GetHistoryAsync(cancellationToken).ConfigureAwait(false); From 845ffe17098b23c74eda2c3fbec528af503d1287 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:08:13 +0100 Subject: [PATCH 12/18] Reword the resilient workflow id health check message State the requirement directly: for resilient workflows every workflow step agent must have an explicit id for crash-recovery resume, and list the step agents to fix. --- .../ResilientWorkflowExecutorIdHealthCheck.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs index f928c0294ac..dc8d6abc775 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ResilientWorkflowExecutorIdHealthCheck.cs @@ -87,6 +87,6 @@ public Task CheckHealthAsync(HealthCheckContext context, Canc return Task.FromResult(new HealthCheckResult( status: context.Registration.FailureStatus, description: - $"Resilient workflow ids: one or more hosted workflow steps use an agent with an auto-generated id, which breaks crash-recovery resume (the checkpoint is matched to the rebuilt workflow by executor id, and an auto-generated id changes on every restart). Assign a stable Id to each agent in the workflow, for example chatClient.AsAIAgent(options: new() {{ Id = \"translator\", Name = \"translator\" }}). Offending steps: {string.Join("; ", offenders)}.")); + $"For resilient workflows, every workflow step agent must have an explicit id so crash-recovery resume can match the checkpoint to the rebuilt workflow (an auto-generated id changes on every restart). Please assign a stable id to each of these step agents: {string.Join("; ", offenders)}.")); } } From 03ee777685c45e05fed6cee05f629c681a700ea6 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:08:14 +0100 Subject: [PATCH 13/18] Update ADR 0032 to match the implemented resilient hosting design Final pass so the ADR reflects what shipped: the recovery bridge seeds from PersistedResponse and skips input re-injection; progress is persisted per output item through the existing FileSystemAgentSessionStore rather than a separate resilient store type; and resilient workflows require stable executor ids, backed by the internal AIAgent.HasAutoGeneratedId signal, the WorkflowHostAgent Workflow service, and a second readiness health check. Implementation-plan items done in this work are marked done and the Python-to-.NET mapping is corrected. --- ...y-hosting-resilient-long-running-agents.md | 65 +++++++++++++------ 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md b/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md index 5c67b0f423b..8a214b274b4 100644 --- a/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md +++ b/docs/decisions/0032-foundry-hosting-resilient-long-running-agents.md @@ -69,9 +69,11 @@ hosting path for no functional gain. ### The recovery contract (same as Python) The platform saves the inputs but not the outputs. After a crash it calls the handler again with the -same input and with `IsRecovery` set to true. The handler rebuilds where it left off from the saved -snapshot only (work that was in flight and not yet saved is left out), and emits an early -`in_progress` event that the client sees as a reset point. On a graceful shutdown the handler calls +same input and with `IsRecovery` set to true. On recovery the handler seeds the outgoing stream from +the last saved snapshot (`PersistedResponse`) instead of the request, so the stream continues from the +output items already sent before the crash rather than re-emitting them, and it does not re-inject the +original input (the restored session already carries the progress, so re-injecting would enqueue a +duplicate turn and the run would never finish). On a graceful shutdown the handler calls `ExitForRecoveryAsync()` to stop cleanly so the platform can restart it later. ### How the handler knows resilience is on (the gate) @@ -112,14 +114,35 @@ pattern already used for the toolbox health check. ### Where the workflow's saved progress lives -A new `FoundryResilientAgentSessionStore` (another implementation of the existing `AgentSessionStore` -base class) owns the resilience-specific persistence: the small pointer to the workflow's last saved -progress per conversation, kept in the platform's per-conversation metadata -(`ConversationChainMetadata`), while the larger workflow state stays in durable storage. The default -store is chosen by configuration: when resilience is on, the registration uses this resilient store; -when resilience is off, it keeps the current `FileSystemAgentSessionStore`. The choice is made lazily -in the store factory, which reads `IOptions` at host startup. A store supplied -explicitly by the developer is always respected. +The handler persists the session at each completed output item, which for a workflow hosted as an +agent is a natural step boundary, by calling `SaveSessionAsync` on the existing `AgentSessionStore`. +The default store is the current `FileSystemAgentSessionStore`, which writes the serialized session +(including the workflow's last checkpoint) under `{HOME}/.checkpoints`, the writable directory the +platform preserves across a restart. On the next invocation `GetSessionAsync` loads that session and +the workflow resumes from its last checkpoint automatically. No separate resilient store type is +needed: saving is gated so it only happens on a resilient background turn (or a recovery turn), and a +store supplied explicitly by the developer is always respected. + +### Resilient workflows require stable executor ids + +A workflow checkpoint records each step by its executor id, and after a crash the workflow is rebuilt +from the same application code and matched to the checkpoint by those ids. An agent-backed step +derives its executor id from the agent's `Id`. When the caller supplies no explicit id, `Id` defaults +to a fresh random value per process, so the rebuilt workflow would use different ids and the resume +would fail deep in the engine with "the specified checkpoint is not compatible with the workflow". + +For resilience to work, every agent used as a workflow step must therefore be given a stable, explicit +id (for example `chatClient.AsAIAgent(options: new() { Id = "translator", Name = "translator" })`). To +turn what would otherwise be a late, cryptic resume failure into an early, actionable signal, a second +readiness health check runs only while resilience is on. It inspects every registered workflow-host +agent and reports unhealthy, before any traffic is routed, when a step's agent has an auto-generated +id. Two small, non-public hooks make this possible without adding to the public API: + +- `AIAgent.HasAutoGeneratedId` (internal): true when no explicit id was supplied (`IdCore` is null). + This is authoritative, so a caller-supplied id that happens to be a GUID is correctly treated as + stable. It is shared with the hosting package through `InternalsVisibleTo`. +- `WorkflowHostAgent.GetService` returns the hosted `Workflow` for an unkeyed `Workflow` request, so + the check can read the workflow's steps and their backing agents without constructing a session. ### Two focused samples @@ -147,14 +170,18 @@ Keeping each sample focused on one behavior makes it easier to demonstrate and t directly and passes it to `AddResponsesServer(...)`. There is no wrapper options type: the Azure SDK `ResponsesServerOptions` already exposes `ResilientBackground` and `SteerableConversations` (along with `DefaultModel`, `DefaultFetchHistoryCount`, and `ResponseAcceptor`). -- **Planned:** the resilience gate, the single-registration guard, and the agent-registration health - check on the registration method. -- **Planned:** the `FoundryResilientAgentSessionStore` with configuration-driven selection. -- **Planned:** the workflow recovery bridge in `AgentFrameworkResponseHandler.CreateAsync`. On - `IsRecovery`, reload the workflow from its last saved progress and continue; save a snapshot at each - step boundary; call `ExitForRecoveryAsync()` on shutdown. +- **Done:** the resilience gate (`ShouldPersistForResilience` plus `IsRecovery`), the single-registration + guard, and the agent-registration health check on the registration method. +- **Done:** the workflow recovery bridge in `AgentFrameworkResponseHandler.CreateAsync`. On + `IsRecovery`, seed the stream from `PersistedResponse` and skip re-injecting input; save the session + at each completed output item so a restart resumes from the last step; call `ExitForRecoveryAsync()` + on shutdown. Persistence uses the existing `FileSystemAgentSessionStore` (no separate resilient store + type was required). +- **Done:** stable executor ids for resilient workflows. `AIAgent.HasAutoGeneratedId` (internal, shared + via `InternalsVisibleTo`) plus `WorkflowHostAgent.GetService()` back a second readiness + health check that fails, while resilience is on, when a workflow step's agent has an auto-generated id. +- **Done:** the two focused samples, each runnable locally with crash-and-recover or steering. - **Planned:** single-agent best-effort behavior plus a guard hook for actions that must not repeat. -- **Planned:** the two focused samples, each runnable locally with crash-and-recover or steering. ## Consequences @@ -173,4 +200,4 @@ Keeping each sample focused on one behavior makes it easier to demonstrate and t | `context.is_recovery` / `context.persisted_response` | `ResponseContext.IsRecovery` / `PersistedResponse` | | `await context.exit_for_recovery()` | `ResponseContext.ExitForRecoveryAsync()` | | `yield stream.checkpoint()` | `ResponseEventStream.Checkpoint()` | -| `Workflow.as_agent(...)` + saved workflow progress | `WorkflowBuilder(...).Build().AsAIAgent(...)` + `FoundryResilientAgentSessionStore` | +| `Workflow.as_agent(...)` + saved workflow progress | `WorkflowBuilder(...).Build().AsAIAgent(...)` + session saved per step in `FileSystemAgentSessionStore` | From 2c00debdde8291b70fc41a7d91460ebd4cff7397 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:08:21 +0100 Subject: [PATCH 14/18] Make the mid-stream resilient session save best-effort A workflow hosted as an agent advances on its own thread, so serializing the session at each completed output item (the resilient checkpoint cadence) can race the running workflow's in-memory checkpoint writes and throw "Collection was modified". Because that save sat in the streaming loop's finally-only try, the exception escaped the handler, the orchestrator logged "Handler failed", and the response was left stuck in_progress forever. On a crash-recovery re-invocation the resumed workflow drains its buffered supersteps in a burst, so the race was effectively guaranteed and recovery never completed. Wrap the incremental save in a try/catch that logs and continues. It is a pure durability optimization: the serialize throws before any file is written, so a failed attempt leaves the previously persisted session intact, and the authoritative save in the finally block (after the stream drains and the workflow is quiescent) persists the final consistent state. A transient snapshot race must never abort the turn. Verified with a local crash-and-recover cycle: a run crashed mid-progress now resumes and reaches status=completed with no "Handler failed" in the restart, where before it stayed in_progress. --- .../AgentFrameworkResponseHandler.cs | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index 676f69f167e..0e1ffad2086 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -503,14 +503,35 @@ await this._toolboxService yield return evt!; // Resilient checkpoint cadence: persist the session at each completed output item, - // a natural workflow phase boundary. Idempotent and gated; awaiting here is safe - // because this sits in the outer (finally-only) try, not the inner try/catch. + // a natural workflow phase boundary. This is a best-effort durability optimization: + // a workflow hosted as an agent runs on its own thread, so serializing the session + // here can overlap with the still-running workflow writing its own in-memory + // checkpoints and throw ("Collection was modified"). That only affects this one + // incremental snapshot, not correctness: the authoritative save in the finally block + // below runs after the stream has fully drained and the workflow has stopped running, + // and persists the final consistent state. The serialize throws before any file is + // written, so a failed attempt leaves the previously persisted session intact. Never + // let such a transient failure abort the turn (which would otherwise escape as an + // unhandled handler failure and leave the response stuck in_progress). if (isResilientTurn && evt is ResponseOutputItemDoneEvent && session is not null && !string.IsNullOrWhiteSpace(sessionConversationId)) { - await sessionStore.SaveSessionAsync(agent, sessionConversationId, session, resolvedUserId, cancellationToken).ConfigureAwait(false); + try + { + await sessionStore.SaveSessionAsync(agent, sessionConversationId, session, resolvedUserId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + if (this._logger.IsEnabled(LogLevel.Debug)) + { + this._logger.LogDebug( + ex, + "Incremental session save was skipped for response {ResponseId}; the end-of-turn save will persist the final state.", + context.ResponseId); + } + } } if (evt is ResponseCompletedEvent or ResponseFailedEvent or ResponseIncompleteEvent) From d2e19a13cca47674d411e7ff6b1248208ae63c93 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:40:31 +0100 Subject: [PATCH 15/18] Add deterministic tests for resilient recovery behavior Drive the handler with a fake agent that records the messages it receives and a fake session store, so the crash-recovery semantics can be asserted without a real model, a real process crash, or timing: - On a recovery re-invocation the handler injects no input (the restored session drives the resume), while a fresh turn feeds the request input to the agent. - A mid-stream session save that throws (the serialize race) is swallowed and the turn still reaches a completed terminal event instead of escaping as a handler failure. --- ...FrameworkResponseHandlerResilienceTests.cs | 260 ++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs new file mode 100644 index 00000000000..a9cee23f782 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs @@ -0,0 +1,260 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.AgentServer.Responses; +using Azure.AI.AgentServer.Responses.Models; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using MeaiTextContent = Microsoft.Extensions.AI.TextContent; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +/// +/// Deterministic tests for the resilient (crash-recovery) behavior of +/// . They drive the handler with a fake agent that +/// records the messages it receives and a fake session store, so recovery semantics can be asserted +/// without a real model, a real process crash, or timing. +/// +public class AgentFrameworkResponseHandlerResilienceTests +{ + [Fact] + public async Task CreateAsync_Recovery_DoesNotReinjectInputAsync() + { + // Arrange: a resilient background+store request being re-invoked as a recovery. + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("original input"); + var context = CreateContext(isRecovery: true); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: on recovery the restored session drives the resume, so the handler must not + // re-inject the original input (which would enqueue a duplicate turn). + Assert.NotNull(recording.LastMessages); + Assert.Empty(recording.LastMessages!); + } + + [Fact] + public async Task CreateAsync_FreshTurn_InjectsInputAsync() + { + // Arrange: the same request on a fresh (non-recovery) turn. + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("original input"); + var context = CreateContext(isRecovery: false); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: a fresh turn feeds the request input to the agent. + Assert.NotNull(recording.LastMessages); + Assert.Contains(recording.LastMessages!, m => m.Text.Contains("original input", StringComparison.Ordinal)); + } + + [Fact] + public async Task CreateAsync_ResilientTurn_MidStreamSaveFailure_StillCompletesAsync() + { + // Arrange: a store whose first save throws, mimicking the serialize race that can happen + // when the incremental (mid-stream) save runs while the workflow is still advancing. The + // later end-of-turn save succeeds. + var store = new ThrowOnceSessionStore(); + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, store, resilient: true); + var request = NewBackgroundStoreRequest("hello"); + var context = CreateContext(isRecovery: false); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: the failed incremental save was swallowed and the turn still reached a completed + // terminal event (it did not escape as a handler failure that leaves the response stuck). + Assert.True(store.SaveAttempts >= 1, "Expected at least one session save attempt."); + Assert.Contains(events, e => e is ResponseCompletedEvent); + Assert.DoesNotContain(events, e => e is ResponseFailedEvent); + } + + [Fact] + public async Task CreateAsync_Recovery_SeedsFromPersistedResponse_DoesNotReemitItsItemsAsync() + { + // Arrange: a recovery whose durable snapshot already carries two output items from the prior + // lifetime. The agent emits one new item on resume. + var persisted = new ResponseObject("resp_" + new string('0', 46), "test"); + persisted.Output.Add(NewMessageItem("prior_1", "prior item one")); + persisted.Output.Add(NewMessageItem("prior_2", "prior item two")); + + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("input"); + var context = CreateContext(isRecovery: true, persistedResponse: persisted); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: the handler seeds the stream from the persisted snapshot. New output items are + // appended starting at output index 2 (the count already in the snapshot), and the two prior + // items are NOT re-emitted as new output items. + var addedIndexes = events.OfType().Select(e => e.OutputIndex).ToList(); + Assert.NotEmpty(addedIndexes); + Assert.All(addedIndexes, i => Assert.True(i >= 2, $"New output item index {i} collided with a seeded item (0 or 1).")); + + // This is exactly why a resilient workflow depends on the snapshot carrying the prior items: + // on resume the workflow does not re-emit already-completed steps (ResumeStreamingInternalAsync + // uses republishPendingEvents: false), so any prior-lifetime output that is not in the + // persisted snapshot never reappears. The end-of-turn snapshot here does carry them, so the + // final response keeps both prior items plus the newly emitted one. + var completed = events.OfType().Single(); + Assert.Equal(3, completed.Response.Output.Count); + } + + private static AgentFrameworkResponseHandler CreateHandler(AIAgent agent, AgentSessionStore store, bool resilient) + { + var services = new ServiceCollection(); + services.AddSingleton(store); + services.AddSingleton(agent); + services.AddSingleton(new FakeHostedSessionIsolationKeyProvider()); + var sp = services.BuildServiceProvider(); + + var options = Options.Create(new ResponsesServerOptions { ResilientBackground = resilient }); + return new AgentFrameworkResponseHandler(sp, NullLogger.Instance, toolboxService: null, responsesServerOptions: options); + } + + private static CreateResponse NewBackgroundStoreRequest(string text) + { + var request = new CreateResponse { Model = "test", Background = true, Store = true }; + request.Input = BinaryData.FromObjectAsJson(new[] + { + new + { + type = "message", + id = "msg_in_1", + status = "completed", + role = "user", + content = new[] { new { type = "input_text", text } } + } + }); + return request; + } + + private static ResponseContext CreateContext(bool isRecovery, ResponseObject? persistedResponse = null) + { + var mock = new Mock("resp_" + new string('0', 46)) { CallBase = true }; + mock.Setup(x => x.IsRecovery).Returns(isRecovery); + mock.Setup(x => x.PersistedResponse).Returns(persistedResponse); + mock.Setup(x => x.GetHistoryAsync(It.IsAny())) + .ReturnsAsync(Array.Empty()); + mock.Setup(x => x.GetInputItemsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + return mock.Object; + } + + private static OutputItemMessage NewMessageItem(string id, string text) => + new( + id: id, + role: MessageRole.Assistant, + content: [new MessageContentOutputTextContent(text, Array.Empty(), Array.Empty())], + status: MessageStatus.Completed); + + private static async Task> CollectEventsAsync( + AgentFrameworkResponseHandler handler, + CreateResponse request, + ResponseContext context) + { + var events = new List(); + await foreach (var evt in handler.CreateAsync(request, context, CancellationToken.None)) + { + events.Add(evt); + } + + return events; + } + + /// + /// A fake agent that records the messages passed to each run so a test can assert exactly what + /// the handler fed it (for example, that recovery injected nothing). + /// + private sealed class RecordingAgent : AIAgent + { + public IReadOnlyList? LastMessages { get; private set; } + + protected override string? IdCore => "recording-agent"; + + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + this.LastMessages = messages.ToList(); + yield return new AgentResponseUpdate + { + MessageId = "msg_rec_1", + Contents = [new MeaiTextContent("recorded")] + }; + await Task.CompletedTask; + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask CreateSessionCoreAsync(CancellationToken cancellationToken = default) => + new(new RecordingSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(JsonSerializer.SerializeToElement(new { }, jsonSerializerOptions)); + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(new RecordingSession()); + + private sealed class RecordingSession : AgentSession + { + public RecordingSession() + { + } + } + } + + /// + /// A fake session store whose first throws (mimicking the + /// serialize race), then succeeds, while loads always create a fresh session. + /// + private sealed class ThrowOnceSessionStore : AgentSessionStore + { + private int _saveAttempts; + + public int SaveAttempts => this._saveAttempts; + + public override ValueTask SaveSessionAsync(AIAgent agent, string conversationId, AgentSession session, string? userId, CancellationToken cancellationToken = default) + { + var attempt = Interlocked.Increment(ref this._saveAttempts); + if (attempt == 1) + { + throw new InvalidOperationException("Collection was modified; enumeration operation may not execute."); + } + + return default; + } + + public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) => + await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + } +} From 471828efe2053e896ba6c27e9588c367e67387d4 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:29:13 +0100 Subject: [PATCH 16/18] Return null from GetSessionAsync and add GetOrCreateSessionAsync for authoritative resume detection GetSessionAsync was a hidden get-or-create that always returned a usable session, so the handler could not tell a resumed conversation from a fresh one and inferred resume from HostedSessionContext, which is only stamped on hosted turns. That regressed local (non-hosted) resumes, replaying history over restored state. GetSessionAsync now returns null when nothing is stored (a plain lookup, never creating), which is the authoritative resume signal in both local and hosted runs and for workflows and chat agents. A new virtual GetOrCreateSessionAsync keeps the convenience path. The handler now derives isResume from whether the store actually loaded a session. --- .../AgentFrameworkResponseHandler.cs | 29 +++---- .../AgentSessionStore.cs | 39 +++++++++- .../FileSystemAgentSessionStore.cs | 6 +- .../InMemoryAgentSessionStore.cs | 13 ++-- ...FrameworkResponseHandlerResilienceTests.cs | 77 ++++++++++++++++++- .../FileSystemAgentSessionStoreTests.cs | 29 +++++-- 6 files changed, 154 insertions(+), 39 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index 0e1ffad2086..0c04eed82bf 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -142,11 +142,20 @@ public override async IAsyncEnumerable CreateAsync( var chatClientAgent = agent.GetService(); - AgentSession? session = !string.IsNullOrWhiteSpace(sessionConversationId) + // Load an existing session when there is a conversation key. The store returns null when + // nothing is persisted for it, which is the authoritative "this is a resume" signal: a + // non-null result means a prior turn saved this session. Whether loaded or created, the + // handler owns creating a fresh session when none exists, so the resume signal does not + // depend on hosted-only bookkeeping and works the same locally and hosted. + AgentSession? loadedSession = !string.IsNullOrWhiteSpace(sessionConversationId) ? await sessionStore.GetSessionAsync(agent, sessionConversationId, resolvedUserId, cancellationToken).ConfigureAwait(false) - : chatClientAgent is not null + : null; + var sessionLoadedFromStore = loadedSession is not null; + + AgentSession? session = loadedSession + ?? (chatClientAgent is not null ? await chatClientAgent.CreateSessionAsync(cancellationToken).ConfigureAwait(false) - : await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + : await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false)); // Capture the platform per-request call id (x-agent-foundry-call-id, protocol 2.0.0 only). // It is re-applied to the ambient HostedCallContext immediately before each outbound egress @@ -155,13 +164,6 @@ public override async IAsyncEnumerable CreateAsync( var platformCallId = context.PlatformContext?.CallId; HostedCallContext.CallId = platformCallId; - // Capture, before the stamping block below writes it, whether this session was already - // established by a prior turn. On a hosted turn the store loads a session that a prior turn - // saved together with its write-once HostedSessionContext; a freshly created session has none - // yet. This specific marker (not a count of state-bag entries) tells a resume from a first - // turn. - var sessionEstablishedByPriorTurn = session?.GetHostedContext() is not null; - // Stamp/validate the hosted identity only when one was resolved. Locally (non-hosted) there is // no user identity, so there is nothing to partition or tamper-check and the session is shared. if (session is not null && resolvedHostedContext is not null) @@ -214,10 +216,11 @@ public override async IAsyncEnumerable CreateAsync( // Load conversation history only for fresh sessions. When a session was already // established by a prior turn (e.g. resuming a workflow paused at an external-input port), // its checkpointed state already contains those messages — replaying history would - // re-drive completed actions and break HITL resume semantics. The signal is the specific - // HostedSessionContext marker captured above, not a count of state-bag entries. + // re-drive completed actions and break HITL resume semantics. The signal is whether the + // store actually loaded a persisted session (sessionLoadedFromStore), which is authoritative + // in both local and hosted runs. var isResume = (!string.IsNullOrWhiteSpace(conversationId) || !string.IsNullOrWhiteSpace(request.PreviousResponseId)) - && sessionEstablishedByPriorTurn; + && sessionLoadedFromStore; if (!isResume) { var history = await context.GetHistoryAsync(cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentSessionStore.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentSessionStore.cs index d1c93dc274c..d507db4d966 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentSessionStore.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentSessionStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -42,7 +43,8 @@ public abstract ValueTask SaveSessionAsync( CancellationToken cancellationToken = default); /// - /// Retrieves a serialized agent session from persistent storage. + /// Retrieves a serialized agent session from persistent storage, or when + /// no session is stored for the given identifiers. /// /// The agent that owns this session. /// The unique identifier for the conversation/session to retrieve. @@ -55,12 +57,41 @@ public abstract ValueTask SaveSessionAsync( /// /// The to monitor for cancellation requests. /// - /// A task that represents the asynchronous retrieval operation. - /// The task result contains the session, or a new session if not found. + /// A task that represents the asynchronous retrieval operation. The task result contains the restored + /// session, or when nothing is stored for the given identifiers. This is a plain + /// lookup: it never creates a session. Use to get a ready-to-use + /// session (loading an existing one or creating a new one), and use this method when the caller needs to + /// distinguish a resumed session from a fresh one (a non-null result means a prior turn established it). /// - public abstract ValueTask GetSessionAsync( + public abstract ValueTask GetSessionAsync( AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default); + + /// + /// Retrieves the stored session for the given identifiers, or creates a new one via + /// when none is stored. + /// + /// The agent that owns this session. + /// The unique identifier for the conversation/session to retrieve. + /// The per-user partition key; see for its meaning. + /// The to monitor for cancellation requests. + /// A task whose result is always a usable session, never . + /// + /// This is the convenience path for callers that only need a session to work with and do not care whether + /// it was loaded or freshly created. It is implemented in terms of , so a + /// store overriding that method gets this behavior for free. + /// + public virtual async ValueTask GetOrCreateSessionAsync( + AIAgent agent, + string conversationId, + string? userId, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(agent); + + return await this.GetSessionAsync(agent, conversationId, userId, cancellationToken).ConfigureAwait(false) + ?? await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + } } diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FileSystemAgentSessionStore.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FileSystemAgentSessionStore.cs index 17c9a184a35..c7c3b7292d4 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FileSystemAgentSessionStore.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FileSystemAgentSessionStore.cs @@ -208,7 +208,7 @@ private string BuildNotWritableMessage(string sessionFilePath) => $"(for example {nameof(InMemoryAgentSessionStore)}) via AddFoundryResponses(agent, agentSessionStore)."; /// - public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) + public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(agent); ArgumentException.ThrowIfNullOrWhiteSpace(conversationId); @@ -216,13 +216,13 @@ public override async ValueTask GetSessionAsync(AIAgent agent, str string path = this.GetSessionPath(agent, conversationId, userId); if (!File.Exists(path)) { - return await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + return null; } byte[] bytes = await File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); if (bytes.Length == 0) { - return await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + return null; } // Parse and clone so the document buffer can be released. diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/InMemoryAgentSessionStore.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/InMemoryAgentSessionStore.cs index 3b3b52fda57..579240432b6 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/InMemoryAgentSessionStore.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/InMemoryAgentSessionStore.cs @@ -40,16 +40,15 @@ public override async ValueTask SaveSessionAsync(AIAgent agent, string conversat } /// - public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) + public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) { var key = GetKey(agent, conversationId, userId); - JsonElement? sessionContent = this._sessions.TryGetValue(key, out var existingSession) ? existingSession : null; - - return sessionContent switch + if (!this._sessions.TryGetValue(key, out var existingSession)) { - null => await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false), - _ => await agent.DeserializeSessionAsync(sessionContent.Value, cancellationToken: cancellationToken).ConfigureAwait(false), - }; + return null; + } + + return await agent.DeserializeSessionAsync(existingSession, cancellationToken: cancellationToken).ConfigureAwait(false); } // Keyed with the same a-/u-/c- prefix scheme as FileSystemAgentSessionStore so the in-memory store diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs index a9cee23f782..3747a19d8c4 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs @@ -61,6 +61,49 @@ public async Task CreateAsync_FreshTurn_InjectsInputAsync() Assert.Contains(recording.LastMessages!, m => m.Text.Contains("original input", StringComparison.Ordinal)); } + [Fact] + public async Task CreateAsync_LocalResume_ExistingStoredSession_SkipsHistoryReplayAsync() + { + // Arrange: a non-recovery turn in local mode (no HostedSessionContext is ever stamped). The + // store already has a session for this conversation, so this is a resume: the restored session + // already carries prior turns, and history must not be replayed on top of it. This is exactly + // the case the previous hosted-only marker got wrong locally. + var recording = new RecordingAgent(); + var store = new StubSessionStore(() => new StubSession()); + var handler = CreateHandler(recording, store, resilient: true); + var request = NewBackgroundStoreRequest("new input"); + request.PreviousResponseId = "resp_prev"; + var context = CreateContext(isRecovery: false, history: [NewMessageItem("hist_1", "PRIOR HISTORY")]); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: the new input is still delivered, but the prior history is not replayed. + Assert.NotNull(recording.LastMessages); + Assert.Contains(recording.LastMessages!, m => m.Text.Contains("new input", StringComparison.Ordinal)); + Assert.DoesNotContain(recording.LastMessages!, m => m.Text.Contains("PRIOR HISTORY", StringComparison.Ordinal)); + } + + [Fact] + public async Task CreateAsync_LocalFresh_NoStoredSession_ReplaysHistoryAsync() + { + // Arrange: the same local, non-recovery turn, but the store has no session yet (a fresh + // conversation). History should be replayed so the first turn sees prior context. + var recording = new RecordingAgent(); + var store = new StubSessionStore(() => null); + var handler = CreateHandler(recording, store, resilient: true); + var request = NewBackgroundStoreRequest("new input"); + request.PreviousResponseId = "resp_prev"; + var context = CreateContext(isRecovery: false, history: [NewMessageItem("hist_1", "PRIOR HISTORY")]); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: with no stored session, this is not a resume, so history is replayed. + Assert.NotNull(recording.LastMessages); + Assert.Contains(recording.LastMessages!, m => m.Text.Contains("PRIOR HISTORY", StringComparison.Ordinal)); + } + [Fact] public async Task CreateAsync_ResilientTurn_MidStreamSaveFailure_StillCompletesAsync() { @@ -145,13 +188,13 @@ private static CreateResponse NewBackgroundStoreRequest(string text) return request; } - private static ResponseContext CreateContext(bool isRecovery, ResponseObject? persistedResponse = null) + private static ResponseContext CreateContext(bool isRecovery, ResponseObject? persistedResponse = null, IReadOnlyList? history = null) { var mock = new Mock("resp_" + new string('0', 46)) { CallBase = true }; mock.Setup(x => x.IsRecovery).Returns(isRecovery); mock.Setup(x => x.PersistedResponse).Returns(persistedResponse); mock.Setup(x => x.GetHistoryAsync(It.IsAny())) - .ReturnsAsync(Array.Empty()); + .ReturnsAsync(history ?? Array.Empty()); mock.Setup(x => x.GetInputItemsAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(Array.Empty()); return mock.Object; @@ -254,7 +297,33 @@ public override ValueTask SaveSessionAsync(AIAgent agent, string conversationId, return default; } - public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) => - await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) + { + await Task.CompletedTask.ConfigureAwait(false); + return null; + } + } + + /// + /// A fake session store that returns whatever (or null) the supplied + /// factory produces, so a test can drive the "session already exists" vs "no session" branch. + /// + private sealed class StubSessionStore : AgentSessionStore + { + private readonly Func _get; + + public StubSessionStore(Func get) + { + this._get = get; + } + + public override ValueTask SaveSessionAsync(AIAgent agent, string conversationId, AgentSession session, string? userId, CancellationToken cancellationToken = default) => default; + + public override ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) => new(this._get()); + } + + /// A minimal concrete used to stand in for a stored session. + private sealed class StubSession : AgentSession + { } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FileSystemAgentSessionStoreTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FileSystemAgentSessionStoreTests.cs index de43efbc5a3..01552edcd15 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FileSystemAgentSessionStoreTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FileSystemAgentSessionStoreTests.cs @@ -50,20 +50,33 @@ public void Constructor_NullOrWhitespaceRoot_Throws() } [Fact] - public async Task GetSessionAsync_NoFileOnDisk_ReturnsFreshSessionFromAgentAsync() + public async Task GetSessionAsync_NoFileOnDisk_ReturnsNullAsync() { var store = new FileSystemAgentSessionStore(this._root); var agent = new TestAgent(); var session = await store.GetSessionAsync(agent, "conv-1", userId: null); + Assert.Null(session); + Assert.Equal(0, agent.CreateCalls); + Assert.Equal(0, agent.DeserializeCalls); + } + + [Fact] + public async Task GetOrCreateSessionAsync_NoFileOnDisk_ReturnsFreshSessionFromAgentAsync() + { + var store = new FileSystemAgentSessionStore(this._root); + var agent = new TestAgent(); + + var session = await store.GetOrCreateSessionAsync(agent, "conv-1", userId: null); + Assert.NotNull(session); Assert.Equal(1, agent.CreateCalls); Assert.Equal(0, agent.DeserializeCalls); } [Fact] - public async Task GetSessionAsync_EmptyFileOnDisk_ReturnsFreshSessionAsync() + public async Task GetSessionAsync_EmptyFileOnDisk_ReturnsNullAsync() { var store = new FileSystemAgentSessionStore(this._root); Directory.CreateDirectory(store.RootDirectory); @@ -72,8 +85,8 @@ public async Task GetSessionAsync_EmptyFileOnDisk_ReturnsFreshSessionAsync() var agent = new TestAgent(); var session = await store.GetSessionAsync(agent, "conv-empty", userId: null); - Assert.NotNull(session); - Assert.Equal(1, agent.CreateCalls); + Assert.Null(session); + Assert.Equal(0, agent.CreateCalls); Assert.Equal(0, agent.DeserializeCalls); } @@ -245,7 +258,7 @@ public async Task GetSessionAsync_NoExistingFile_DoesNotCreateAgentDirectoryAsyn var session = await store.GetSessionAsync(agent, "missing-id", userId: null); - Assert.NotNull(session); + Assert.Null(session); Assert.False(Directory.Exists(this._root), "Read miss must not create the root directory."); } @@ -385,11 +398,11 @@ public async Task GetSessionAsync_DifferentUser_DoesNotReadAnotherUsersSessionAs await store.SaveSessionAsync(agent, "shared-conv", NewSession(), userId: "alice"); // Bob requests the same conversationId. The per-user partition means Bob's path is distinct, - // so the store returns a fresh session (no leak), not Alice's persisted state. + // so the store returns null (no leak), not Alice's persisted state. var bobSession = await store.GetSessionAsync(agent, "shared-conv", userId: "bob"); - Assert.NotNull(bobSession); - Assert.Equal(1, agent.CreateCalls); // fresh session created for Bob + Assert.Null(bobSession); // no session for Bob under his partition + Assert.Equal(0, agent.CreateCalls); // a plain lookup never creates Assert.Equal(0, agent.DeserializeCalls); // Alice's file never deserialized for Bob } From 0602895c9d23c93ddfe06ea52095307a53ca757c Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:29:24 +0100 Subject: [PATCH 17/18] Resolve agents via IServiceProvider so the resilient id health check covers factory registrations The readiness health check read AIAgent registrations straight from the service descriptors, which only exposes agents registered as a concrete instance. Agents registered via a factory (including keyed factory registrations, the documented way to host multiple workflow agents) were never inspected, so their executor ids went unchecked. Resolving from the service provider instead covers both instance and factory registrations. Keys are discovered from the descriptors because a service provider cannot enumerate keyed registrations on its own; agents are singletons so each is constructed at most once and cached. --- .../ServiceCollectionExtensions.cs | 32 +++++++++------ .../ServiceCollectionExtensionsTests.cs | 39 +++++++++++++++++++ 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs index 94282bcb466..665b6a5de02 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs @@ -382,31 +382,41 @@ private static void AddResilientWorkflowIdHealthCheck(IServiceCollection service name: HealthCheckName, factory: sp => new ResilientWorkflowExecutorIdHealthCheck( isResilient: () => sp.GetService>()?.Value.ResilientBackground ?? false, - agents: () => EnumerateRegisteredAgents(services)), + agents: () => ResolveRegisteredAgents(sp, services)), failureStatus: HealthStatus.Unhealthy, tags: ["foundry", "workflow", "readiness"])); }); } /// - /// Enumerates the instances registered on as - /// concrete singletons (default or keyed). Reads the descriptors directly so no agent is - /// resolved or constructed; agents registered via a factory (rather than an instance) are not - /// inspectable this way and are skipped. + /// Resolves the registered instances (default and keyed) from + /// . Resolving (rather than reading descriptors) is what lets the + /// health check see agents registered via a factory, not just those registered as a concrete + /// instance. The keys are discovered from because a + /// cannot enumerate keyed registrations on its own. Agents are + /// singletons here, so each is constructed at most once and cached for later readiness polls. /// - private static IEnumerable EnumerateRegisteredAgents(IServiceCollection services) + private static IEnumerable ResolveRegisteredAgents(IServiceProvider serviceProvider, IServiceCollection services) { + // Default (non-keyed) registrations. + foreach (var agent in serviceProvider.GetServices()) + { + yield return agent; + } + + // Keyed registrations: discover each distinct key from the descriptors, then resolve it. + var seenKeys = new HashSet(); foreach (var descriptor in services) { - if (descriptor.ServiceType != typeof(AIAgent)) + if (descriptor.ServiceType != typeof(AIAgent) + || !descriptor.IsKeyedService + || descriptor.ServiceKey is null + || !seenKeys.Add(descriptor.ServiceKey)) { continue; } - object? instance = descriptor.IsKeyedService - ? descriptor.KeyedImplementationInstance - : descriptor.ImplementationInstance; - if (instance is AIAgent agent) + foreach (var agent in serviceProvider.GetKeyedServices(descriptor.ServiceKey)) { yield return agent; } diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs index 0e0a484dc62..854b8f9e8f2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs @@ -5,6 +5,7 @@ using System.Net; using System.Threading.Tasks; using Azure.AI.AgentServer.Responses; +using Microsoft.Agents.AI.Workflows; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.TestHost; @@ -109,6 +110,44 @@ public void AddFoundryResponses_RegistersAgentRegistrationHealthCheck() Assert.Contains(options.Value.Registrations, r => r.Name == "foundry-agent-registration"); } + [Fact] + public async Task ResilientWorkflowIdHealthCheck_SeesFactoryRegisteredAgentAsync() + { + // Arrange: resilient host with a workflow agent registered via a KEYED FACTORY (not a concrete + // instance). Its step agents have auto-generated ids, which would break crash-recovery resume. + var services = new ServiceCollection(); + services.AddLogging(); + services.AddFoundryResponses(o => o.ResilientBackground = true); + services.AddKeyedSingleton("wf", (sp, key) => + { + var french = new StreamingTextAgent("french", "bonjour"); + var spanish = new StreamingTextAgent("spanish", "hola"); + return new WorkflowBuilder(french).AddEdge(french, spanish).Build().AsAIAgent(name: "wf-host"); + }); + + var provider = services.BuildServiceProvider(); + var registration = provider + .GetRequiredService>() + .Value.Registrations.Single(r => r.Name == "foundry-resilient-workflow-ids"); + + var check = registration.Factory(provider); + var context = new Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckContext + { + Registration = new Microsoft.Extensions.Diagnostics.HealthChecks.HealthCheckRegistration( + registration.Name, + check, + Microsoft.Extensions.Diagnostics.HealthChecks.HealthStatus.Unhealthy, + tags: null), + }; + + // Act: resolving (not descriptor-reading) is what lets the check see a factory-registered agent. + var result = await check.CheckHealthAsync(context); + + // Assert + Assert.Equal(Microsoft.Extensions.Diagnostics.HealthChecks.HealthStatus.Unhealthy, result.Status); + } + [Fact] public void AddFoundryResponses_NullServices_ThrowsArgumentNullException() { From 17be570a14bb0e5a2695c659764bd2a7d0ce04d8 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:54:48 +0100 Subject: [PATCH 18/18] Bump Azure.AI.AgentServer preview packages to Core beta.28, Invocations beta.7, Responses beta.8 Refreshed local preview drop from the long-running-agents-private-preview main branch. Foundry.Hosting builds clean and the full unit suite stays green (345/345) on the new surface. --- dotnet/Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index ecebc889c9a..b76ece41fce 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -23,9 +23,9 @@ - - - + + +