From ae543aff33c603a586f57f63c887696afdb415df Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 20:17:00 +0000 Subject: [PATCH 01/11] Report only the exit status of a failed sandboxed initialization step The shared Runtime initializer now adds exit_code to its failed receipt when a setup command or package manager run through bwrap exits nonzero. It never serializes the exception, command, input or child output; Runtime-internal helpers, signals and other errors keep the generic receipt. --- Makefile | 1 + .../agents-api/deploy/runtime/initialize.py | 21 ++++- .../deploy/runtime/initialize_receipt_test.py | 81 +++++++++++++++++++ .../deploy/runtime/initialize_test.py | 17 ++-- 4 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 services/agents-api/deploy/runtime/initialize_receipt_test.py diff --git a/Makefile b/Makefile index 8b3e1f449..e57fb6309 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ build-agents-api-release: check-agents-api: build-agents-api go test ./services/agents-api/... ./packages/agents-client/... -count=1 + PYTHONDONTWRITEBYTECODE=1 python3 services/agents-api/deploy/runtime/initialize_receipt_test.py docker-build-agents-api: ./scripts/build-agents-api-image.sh diff --git a/services/agents-api/deploy/runtime/initialize.py b/services/agents-api/deploy/runtime/initialize.py index 04a492293..77fd35f6a 100644 --- a/services/agents-api/deploy/runtime/initialize.py +++ b/services/agents-api/deploy/runtime/initialize.py @@ -218,6 +218,22 @@ def run(request): stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) +def failed_receipt(error): + """Report only the integer exit status of a step run inside the isolation sandbox. + + Setup commands and package managers (including apt from the system tool root) + run through bwrap. Runtime-internal helpers such as seed extraction and the + Skill writer do not, so their failures stay generic. The exception itself is + never serialized: it can contain the command, input or captured output. + """ + receipt = {'version': 1, 'outcome': 'failed'} + code = getattr(error, 'returncode', None) + if (isinstance(error, subprocess.CalledProcessError) and isinstance(error.cmd, list) + and error.cmd[:1] == ['/usr/bin/bwrap'] and type(code) is int and 0 < code < 256): + receipt['exit_code'] = code + return json.dumps(receipt, separators=(',', ':')) + + def stdio_lifetime(args): """Bind sandbox lifetime to the native process, not its transient spawn thread.""" parent = os.getppid() @@ -297,9 +313,8 @@ def main(): raise ValueError('invalid version') roots() run(request) - except Exception: - # Never serialize an exception that could contain input or process args. - print('{"version":1,"outcome":"failed"}') + except Exception as error: + print(failed_receipt(error)) return 1 print('{"version":1,"outcome":"completed"}') return 0 diff --git a/services/agents-api/deploy/runtime/initialize_receipt_test.py b/services/agents-api/deploy/runtime/initialize_receipt_test.py new file mode 100644 index 000000000..b7c07bad6 --- /dev/null +++ b/services/agents-api/deploy/runtime/initialize_receipt_test.py @@ -0,0 +1,81 @@ +"""Host-runnable checks of the initializer's failure receipt. + +The receipt may carry only the integer exit status of a sandboxed step, never its +command, input or output. Real isolation is covered by initialize_test.py inside a +packaged Runtime; these checks need no Runtime and run with plain python3. +""" +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import unittest +from unittest import mock + +SPEC = importlib.util.spec_from_file_location('agents_api_runtime_initialize', Path(__file__).with_name('initialize.py')) +initialize = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(initialize) +CANARY = 'CANARY-initializer-receipt-5d1c' +SANDBOXED = ['/usr/bin/bwrap', '--unshare-user', '--', '/bin/bash', '-c', 'echo ' + CANARY + '; exit 3'] + + +def invoke(failure): + """Run main() with a request whose step raises failure; return (code, stdout, stderr).""" + request = json.dumps({'version': 1, 'action': 'setup', 'network': 'enabled', 'command': 'echo ' + CANARY}) + stdin = mock.Mock() + stdin.buffer = io.BytesIO(request.encode()) + stdout, stderr = io.StringIO(), io.StringIO() + with mock.patch.object(initialize.sys, 'stdin', stdin), mock.patch.object(initialize.sys, 'argv', ['initialize']), \ + mock.patch.object(initialize, 'roots', lambda: None), mock.patch.object(initialize, 'run', side_effect=failure), \ + contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = initialize.main() + return code, stdout.getvalue(), stderr.getvalue() + + +class FailureReceiptTest(unittest.TestCase): + def assert_receipt(self, failure, expected): + code, stdout, stderr = invoke(failure) + self.assertEqual(code, 1) + self.assertEqual(stderr, '') + self.assertEqual(json.loads(stdout), expected) + self.assertNotIn(CANARY, stdout) + self.assertNotIn('echo', stdout) + + def test_sandboxed_step_reports_only_its_exit_status(self): + for status in (1, 3, 100, 255): + failure = subprocess.CalledProcessError(status, SANDBOXED, output=CANARY.encode(), stderr=CANARY.encode()) + self.assert_receipt(failure, {'version': 1, 'outcome': 'failed', 'exit_code': status}) + # The receipt keeps its compact, stable encoding. + _, stdout, _ = invoke(subprocess.CalledProcessError(3, SANDBOXED)) + self.assertEqual(stdout, '{"version":1,"outcome":"failed","exit_code":3}\n') + + def test_runtime_helpers_signals_and_other_errors_stay_generic(self): + generic = {'version': 1, 'outcome': 'failed'} + for failure in ( + subprocess.CalledProcessError(2, ['/usr/bin/tar', '-xzf', CANARY]), + subprocess.CalledProcessError(1, ['/usr/local/bin/agents-api-codex-write', CANARY]), + subprocess.CalledProcessError(-9, SANDBOXED), + subprocess.CalledProcessError(256, SANDBOXED), + subprocess.CalledProcessError(3, ' '.join(SANDBOXED)), + subprocess.TimeoutExpired(SANDBOXED, 120, output=CANARY.encode()), + ValueError(CANARY), + OSError(CANARY), + ): + with self.subTest(failure=type(failure).__name__): + self.assert_receipt(failure, generic) + _, stdout, _ = invoke(ValueError(CANARY)) + self.assertEqual(stdout, '{"version":1,"outcome":"failed"}\n') + + def test_real_child_output_never_reaches_the_receipt(self): + # A real process failure, launched like run() with discarded output. + try: + subprocess.run(['/bin/sh', '-c', 'echo ' + CANARY + '; echo ' + CANARY + ' >&2; exit 7'], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + except subprocess.CalledProcessError as error: + failure = subprocess.CalledProcessError(error.returncode, ['/usr/bin/bwrap', '--', *error.cmd]) + self.assert_receipt(failure, {'version': 1, 'outcome': 'failed', 'exit_code': 7}) + + +if __name__ == '__main__': + unittest.main() diff --git a/services/agents-api/deploy/runtime/initialize_test.py b/services/agents-api/deploy/runtime/initialize_test.py index 70e19dbea..616ce6680 100644 --- a/services/agents-api/deploy/runtime/initialize_test.py +++ b/services/agents-api/deploy/runtime/initialize_test.py @@ -16,14 +16,17 @@ CANARY = 'private-initialization-canary-47a8' -def invoke(action, *, succeeds=True, **fields): +def invoke(action, *, succeeds=True, exit_code=None, **fields): payload = json.dumps({'version': 1, 'action': action, 'network': 'enabled', **fields}) result = subprocess.run(['/usr/bin/python3', '-I', '-S', HELPER], input=payload, text=True, capture_output=True, timeout=120) - expected = 'completed' if succeeds else 'failed' + expected = {'version': 1, 'outcome': 'completed' if succeeds else 'failed'} + if exit_code is not None: + # A failed sandboxed step reports only its exit status, never its output. + expected['exit_code'] = exit_code assert result.returncode == (0 if succeeds else 1), (action, result.returncode) assert result.stderr == '', (action, 'unexpected stderr') - assert json.loads(result.stdout) == {'version': 1, 'outcome': expected}, action + assert json.loads(result.stdout) == expected, (action, result.stdout) assert CANARY not in result.stdout + result.stderr, 'confidential output exposed' @@ -100,8 +103,10 @@ def main(): assert result.stdout == cwd + '\n42\n', result.stdout invoke('setup', cwd='/workspace/sub', command='test -f ../first && pwd > second') assert Path('/environment/workspace/sub/second').read_text() == '/workspace/sub\n' - invoke('setup', succeeds=False, command='echo secret; echo secret >&2; exit 7') - invoke('setup', succeeds=False, cwd='/missing', command='touch /workspace/should-not-exist') + invoke('setup', succeeds=False, exit_code=7, command='echo secret; echo secret >&2; exit 7') + invoke('setup', succeeds=False, exit_code=3, command='echo "$INITIALIZATION_VALUE"; echo "$INITIALIZATION_VALUE" >&2; exit 3') + # bwrap reports its own failure to enter the missing cwd as status 1. + invoke('setup', succeeds=False, exit_code=1, cwd='/missing', command='touch /workspace/should-not-exist') assert not Path('/environment/workspace/should-not-exist').exists() invoke('setup', command='setsid /bin/bash -c "sleep 2; touch /workspace/descendant" >/dev/null 2>&1 &') time.sleep(3) @@ -110,6 +115,8 @@ def main(): # Actual public registries, not synthetic package fixtures. invoke('npm', packages=['is-number@7.0.0']) invoke('python', packages=['packaging==26.0']) + # pip's diagnostics name the package and can echo configuration; only its status is reported. + invoke('python', succeeds=False, exit_code=1, packages=['parsar-initializer-nonexistent-4f7e-zz']) invoke('setup', cwd='/workspace/sub', command="node -e \"if (!require('/environment/packages/npm/lib/node_modules/is-number')(42)) process.exit(1)\" && python3 -c 'import packaging; assert packaging.__version__ == \"26.0\"'") print(json.dumps({'initialization': 'passed', 'real_packages': '--packages' in sys.argv, 'system_packages': '--system' in sys.argv})) From f719ae3792b265d128e36a64b697c316191b2a40 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 20:47:24 +0000 Subject: [PATCH 02/11] Fail the Session when its hosted Environment fails to provision A hosted initialization failure now records, in the allocation's cleanup transaction, the failed Environment with a safe reason and failure time (additive nullable migration 000061), agent.session.environment.failed with the observed environment_error/environment_connection_failed payload, an error event (environment_error/sandbox_error, param null) and one agent.session.failed snapshot. Session retrieve, list and events derive status failed, the reason and last_active_at from that record; GET and creation streams end after the failed event; new input returns the observed 409 conflict_error. Pending-input settlement, expiry, self_hosted and none are unchanged. The reason names only a fixed step label and the Runtime-reported exit status of a confirmed failed step; unknown effects, timeouts and receipts without an exit status use a generic reason. Command, package-manager and file output is never copied. --- contracts/agents-api/openapi.yaml | 59 +-- contracts/agents-api/v1/events.go | 21 +- contracts/agents-api/v1/events_test.go | 24 ++ services/agents-api/internal/api/errors.go | 3 + .../internal/api/hosted_failure_test.go | 194 +++++++++ services/agents-api/internal/api/inputs.go | 2 +- .../internal/api/session_response.go | 10 + services/agents-api/internal/api/stream.go | 15 +- .../db/queries/environment_connections.sql | 5 + .../db/sqlc/environment_connections.sql.go | 18 + .../internal/db/sqlc/environments.sql.go | 8 +- .../agents-api/internal/db/sqlc/models.go | 10 +- .../execution/runtime_initialization.go | 28 +- .../internal/execution/runtime_setup.go | 43 +- .../internal/store/environment_connections.go | 34 +- .../internal/store/environment_inputs.go | 10 + .../store/environments_migration_test.go | 8 +- ...sted_initialization_failure_public_test.go | 384 ++++++++++++++++++ .../store/runtime_allocation_state.go | 21 +- .../store/runtime_environment_terminal.go | 128 +++++- .../runtime_environment_terminal_test.go | 54 ++- .../agents-api/internal/store/scheduling.go | 1 + .../internal/store/session_events.go | 3 + .../agents-api/internal/store/sessions.go | 3 + .../migrations/000061_environment_failure.sql | 18 + 25 files changed, 1027 insertions(+), 77 deletions(-) create mode 100644 services/agents-api/internal/api/hosted_failure_test.go create mode 100644 services/agents-api/internal/store/hosted_initialization_failure_public_test.go create mode 100644 services/agents-api/migrations/000061_environment_failure.sql diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index b287c9f75..1b2af9c62 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -2481,6 +2481,12 @@ definitions: type: string message: type: string + param: + description: |- + Param is the pinned SessionError field. A top-level error event always + carries it, null when unset; Environment state errors omit it. + type: string + x-nullable: true type: type: string type: object @@ -4369,9 +4375,12 @@ paths: indexes. Native text conversion and output quotas apply; completion snapshots remain authoritative. Reconnect through Session, Turn and Items reads; missed events are not replayed. A lagging stream closes with an error when its bounded - buffer is exceeded. Session activity includes immutable pending-input connection - actions before Turn creation; self_hosted environments use the same safe output - as Session retrieval. + buffer is exceeded. When a hosted Environment fails to provision, the stream + sends agent.session.environment.failed, an error event (environment_error/sandbox_error + with the safe step and exit-status reason, never command output) and agent.session.failed, + then ends. Session activity includes immutable pending-input connection actions + before Turn creation; self_hosted environments use the same safe output as + Session retrieval. parameters: - description: agents=v1 in: header @@ -4434,27 +4443,29 @@ paths: batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled - errors; exact hosted failure mapping is unverified. Input the Session cannot - accept in its current state, such as a result after cancellation or a batch - while earlier input is pending, and a result that differs from the call's - saved result return 409 with type and code conflict_error; reusing an Idempotency-Key - with a different batch returns the local 409 idempotency_conflict. Inside - an owned Session, a result for an unknown call or for a call of another Turn - returns 400 invalid_request_error and changes nothing; missing and foreign - Sessions return 404. Losing execution ownership returns 503. The response - write deadline accommodates the admission window for either prepared Environment, - independently of new-hosted-admission and executor URL settings. Disconnecting - the waiting HTTP request does not cancel retained work or restart its deadline. - Retry keys identify the whole ordered batch. Function output accepts text - or ordered text/image parts subject to engine support; Claude SDK accepts - text results and, on none and qualified openai_hosted, successful inline PNG/JPEG - results, preserving ordered content; error images and remote references reject - before admission. Native image resizing may change bytes. Runtime image-result - support is checked only for image-bearing delivery. Codex and Claude SDK on - none and qualified openai_hosted accept ordered inline PNG/JPEG image messages. - Self-hosted profiles and other engines remain text-only; remote image URLs - are unsupported. Image references are retained unchanged without service-side - downloads. + errors. New input on a Session whose hosted Environment failed to provision + returns the observed 409 conflict_error "the hosted environment failed to + provision"; input already waiting when it fails and expired Environments keep + the local 409 environment_unavailable. Input the Session cannot accept in + its current state, such as a result after cancellation or a batch while earlier + input is pending, and a result that differs from the call's saved result return + 409 with type and code conflict_error; reusing an Idempotency-Key with a different + batch returns the local 409 idempotency_conflict. Inside an owned Session, + a result for an unknown call or for a call of another Turn returns 400 invalid_request_error + and changes nothing; missing and foreign Sessions return 404. Losing execution + ownership returns 503. The response write deadline accommodates the admission + window for either prepared Environment, independently of new-hosted-admission + and executor URL settings. Disconnecting the waiting HTTP request does not + cancel retained work or restart its deadline. Retry keys identify the whole + ordered batch. Function output accepts text or ordered text/image parts subject + to engine support; Claude SDK accepts text results and, on none and qualified + openai_hosted, successful inline PNG/JPEG results, preserving ordered content; + error images and remote references reject before admission. Native image resizing + may change bytes. Runtime image-result support is checked only for image-bearing + delivery. Codex and Claude SDK on none and qualified openai_hosted accept + ordered inline PNG/JPEG image messages. Self-hosted profiles and other engines + remain text-only; remote image URLs are unsupported. Image references are + retained unchanged without service-side downloads. parameters: - description: agents=v1 in: header diff --git a/contracts/agents-api/v1/events.go b/contracts/agents-api/v1/events.go index 476557c49..3f6f41b40 100644 --- a/contracts/agents-api/v1/events.go +++ b/contracts/agents-api/v1/events.go @@ -29,6 +29,9 @@ type StreamError struct { Code string `json:"code"` Type string `json:"type"` Message string `json:"message"` + // Param is the pinned SessionError field. A top-level error event always + // carries it, null when unset; Environment state errors omit it. + Param *string `json:"param,omitempty" extensions:"x-nullable"` } // TerminalTurnEvent reports whether an event type settles a Turn. @@ -45,11 +48,27 @@ func itemEvent(eventType string) bool { return eventType == "agent.session.turn.item.added" || eventType == "agent.session.turn.item.done" } +// sessionError is the pinned SessionError of a top-level error event, whose +// param is present and null when unset (HI-01). +type sessionError struct { + Code string `json:"code"` + Type string `json:"type"` + Message string `json:"message"` + Param *string `json:"param"` +} + // MarshalJSON keeps the nullable top-level usage on terminal Turn events only, -// and a nullable output_index on every Item event (EVT-09). +// a nullable output_index on every Item event (EVT-09) and a nullable error +// param on error events. func (e SessionEvent) MarshalJSON() ([]byte, error) { type wire SessionEvent switch { + case e.Type == "error" && e.Error != nil: + e.Usage = nil + return json.Marshal(struct { + wire + Error sessionError `json:"error"` + }{wire(e), sessionError(*e.Error)}) case TerminalTurnEvent(e.Type): return json.Marshal(struct { wire diff --git a/contracts/agents-api/v1/events_test.go b/contracts/agents-api/v1/events_test.go index 5d0891072..05ac64fdb 100644 --- a/contracts/agents-api/v1/events_test.go +++ b/contracts/agents-api/v1/events_test.go @@ -42,3 +42,27 @@ func TestSessionEventUsageOnlyOnTerminalTurnEvents(t *testing.T) { } } } + +// Error events carry the pinned SessionError, whose param is present and null +// when unset; Environment state errors keep their observed three fields (HI-01/02). +func TestSessionErrorEventCarriesNullableParam(t *testing.T) { + failure := &StreamError{Type: "environment_error", Code: "sandbox_error", Message: "Failed to provision environment"} + param := "input" + for _, test := range []struct { + event SessionEvent + want string + }{ + {SessionEvent{Type: "error", EventID: "event", SessionID: "session", Error: failure}, + `{"type":"error","event_id":"event","session_id":"session","error":{"code":"sandbox_error","type":"environment_error","message":"Failed to provision environment","param":null}}`}, + {SessionEvent{Type: "error", EventID: "event", SessionID: "session", Error: &StreamError{Type: "invalid_request_error", Code: "invalid", Message: "m", Param: ¶m}}, + `{"type":"error","event_id":"event","session_id":"session","error":{"code":"invalid","type":"invalid_request_error","message":"m","param":"input"}}`}, + {SessionEvent{Type: "agent.session.environment.failed", EventID: "event", Environment: &SessionEnvironmentState{ID: "environment", Type: "openai_hosted", Status: "failed", + Error: &StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."}}}, + `{"type":"agent.session.environment.failed","event_id":"event","environment":{"id":"environment","type":"openai_hosted","status":"failed","error":{"code":"environment_connection_failed","type":"environment_error","message":"The environment failed to connect."}}}`}, + } { + raw, err := json.Marshal(test.event) + if err != nil || string(raw) != test.want { + t.Fatalf("%s %v", raw, err) + } + } +} diff --git a/services/agents-api/internal/api/errors.go b/services/agents-api/internal/api/errors.go index e9ddcccf2..f40e1ba5e 100644 --- a/services/agents-api/internal/api/errors.go +++ b/services/agents-api/internal/api/errors.go @@ -107,6 +107,9 @@ func writeStoreError(w http.ResponseWriter, r *http.Request, err error, notFound writeError(w, http.StatusRequestEntityTooLarge, "request_too_large", "File exceeds this operation's content limit.") case errors.Is(err, store.ErrCredentialStorageUnavailable): writeError(w, http.StatusServiceUnavailable, "credential_storage_unavailable", "Credential encryption is not configured on this service.") + case errors.Is(err, store.ErrHostedEnvironmentFailed): + // Observed official status, type, code, null param and message. + writeError(w, http.StatusConflict, "conflict_error", "the hosted environment failed to provision") case errors.Is(err, store.ErrEnvironmentUnavailable): writeError(w, http.StatusConflict, "environment_unavailable", "The environment is no longer available for new input.") case errors.Is(err, execution.ErrEnvironmentInputExpired): diff --git a/services/agents-api/internal/api/hosted_failure_test.go b/services/agents-api/internal/api/hosted_failure_test.go new file mode 100644 index 000000000..ddc351005 --- /dev/null +++ b/services/agents-api/internal/api/hosted_failure_test.go @@ -0,0 +1,194 @@ +package api + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +const hostedFailureReason = `Failed to provision environment: script "setup_commands[0]" failed with exit code 3` + +func hostedFailureSession() store.Session { + session := environmentSession() + session.Configuration = json.RawMessage(`{"agent":{"id":"agent_test","model":"model","tools":[]},"environment":{"type":"openai_hosted"}}`) + session.Environment.Configuration = json.RawMessage(`{"type":"openai_hosted"}`) + session.Environment.Status = "failed" + return session +} + +// A recorded hosted provisioning failure makes the Session failed with the safe +// reason and failure time, whether or not pending input was settled with it. +func TestHostedProvisioningFailureSessionProjection(t *testing.T) { + failedAt := time.Unix(1790187038, 0) + for _, activity := range []*store.EnvironmentInputActivity{nil, {Status: "failed", Failure: "environment_unavailable", LastActiveAt: failedAt.Add(-time.Second)}} { + session := hostedFailureSession() + session.EnvironmentInputActivity = activity + session.EnvironmentFailure = &store.EnvironmentFailure{Reason: hostedFailureReason, FailedAt: failedAt} + response, err := sessionResponse(session, "") + if err != nil || response.Status != "failed" || response.Error == nil || *response.Error != hostedFailureReason || + response.LastActiveAt != failedAt.Unix() || response.RequiredActions == nil || len(response.RequiredActions) != 0 { + t.Fatal("failed hosted Session projection", response, err) + } + } + // Without a recorded failure (expiry, or failures before this release), the + // existing projection is unchanged. + session := hostedFailureSession() + if response, err := sessionResponse(session, ""); err != nil || response.Status != "idle" || response.Error != nil { + t.Fatal("unrecorded failure changed the projection", response, err) + } + session = environmentSession() + session.EnvironmentFailure = &store.EnvironmentFailure{Reason: hostedFailureReason, FailedAt: failedAt} + if _, err := sessionResponse(session, environmentOrigin); err == nil { + t.Fatal("self-hosted Session accepted a hosted provisioning failure") + } +} + +// The error event carries the pinned SessionError with a null param; the +// Environment state error keeps the observed three fields. +func TestHostedProvisioningFailureEventShapes(t *testing.T) { + session := hostedFailureSession() + fields := func(change store.SessionChange) map[string]any { + t.Helper() + event, err := streamResponse(session, change, "") + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(raw, &value); err != nil { + t.Fatal(err) + } + return value + } + got := fields(store.SessionChange{Event: v1.SessionEvent{Type: "error", EventID: "event", SessionID: "session", + Error: &v1.StreamError{Type: "environment_error", Code: "sandbox_error", Message: hostedFailureReason}}}) + want := map[string]any{"type": "error", "event_id": "event", "session_id": "session", + "error": map[string]any{"type": "environment_error", "code": "sandbox_error", "message": hostedFailureReason, "param": nil}} + if !reflect.DeepEqual(got, want) { + t.Fatal("error event", got) + } + got = fields(store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.environment.failed", EventID: "event", SessionID: "session", + Environment: &v1.SessionEnvironmentState{ID: "environment", Type: "openai_hosted", Status: "failed", + Error: &v1.StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."}}}}) + if environment, _ := got["environment"].(map[string]any); !reflect.DeepEqual(environment["error"], map[string]any{ + "type": "environment_error", "code": "environment_connection_failed", "message": "The environment failed to connect."}) { + t.Fatal("environment.failed event", got) + } + failedAt := time.Unix(1790187038, 0) + got = fields(store.SessionChange{Event: v1.SessionEvent{Type: "agent.session.failed", EventID: "event"}, + EnvironmentFailure: &store.EnvironmentFailure{Reason: hostedFailureReason, FailedAt: failedAt}}) + if snapshot, _ := got["session"].(map[string]any); snapshot["status"] != "failed" || snapshot["error"] != hostedFailureReason || + snapshot["last_active_at"] != float64(failedAt.Unix()) || !reflect.DeepEqual(snapshot["required_actions"], []any{}) { + t.Fatal("failed snapshot", got) + } +} + +// H5: a GET stream ends after the agent.session.failed of a hosted provisioning +// failure; a Turn failure leaves it open because the Session can continue. +func TestGetStreamEndsAfterHostedProvisioningFailure(t *testing.T) { + for _, terminal := range []bool{true, false} { + t.Run(fmt.Sprint(terminal), func(t *testing.T) { + f := &streamFixture{session: hostedFailureSession()} + f.session.TenantID = uuid.NewString() + f.session.Environment.TenantID = f.session.TenantID + auth, err := NewAuthenticator([]APIKey{{OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: device.HashCredential("key"), TenantID: f.session.TenantID}}) + if err != nil { + t.Fatal(err) + } + h, err := NewHandler(f, auth, "codex") + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(h) + defer server.Close() + failed := store.SessionChange{Sequence: 13, Event: v1.SessionEvent{Type: "agent.session.failed", EventID: "failed"}} + if terminal { + failed.EnvironmentFailure = &store.EnvironmentFailure{Reason: hostedFailureReason, FailedAt: time.Unix(1790187038, 0)} + } else { + failed.Turn = &store.Turn{ID: "turn", Status: store.TurnFailed} + } + f.changes = []store.SessionChange{ + {Sequence: 11, Event: v1.SessionEvent{Type: "agent.session.environment.failed", EventID: "environment", SessionID: "session", + Environment: &v1.SessionEnvironmentState{ID: "environment", Type: "openai_hosted", Status: "failed", Error: &v1.StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."}}}}, + {Sequence: 12, Event: v1.SessionEvent{Type: "error", EventID: "error", SessionID: "session", Error: &v1.StreamError{Type: "environment_error", Code: "sandbox_error", Message: hostedFailureReason}}}, + failed, + } + request, _ := http.NewRequest(http.MethodGet, server.URL+"/v1/agents/sessions/session/events", nil) + request.Header.Set("Authorization", "Bearer key") + request.Header.Set("OpenAI-Beta", "agents=v1") + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + lines := make(chan string, 32) + go func() { + defer close(lines) + scanner := bufio.NewScanner(response.Body) + scanner.Buffer(nil, 1<<20) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "event: ") { + lines <- strings.TrimPrefix(scanner.Text(), "event: ") + } + } + }() + var names []string + for len(names) < 3 { + select { + case name := <-lines: + names = append(names, name) + case <-time.After(5 * time.Second): + t.Fatal("events not delivered", names) + } + } + if !reflect.DeepEqual(names, []string{"agent.session.environment.failed", "error", "agent.session.failed"}) { + t.Fatal("event order", names) + } + select { + case _, open := <-lines: + if open || !terminal { + t.Fatal("stream lifetime after failure", open, terminal) + } + case <-time.After(1500 * time.Millisecond): + if terminal { + t.Fatal("GET stream stayed open after the terminal failure") + } + } + }) + } +} + +// H6: new input on a failed hosted Environment gets the observed 409; expiry +// keeps the local environment_unavailable response. +func TestHostedProvisioningFailureInputConflict(t *testing.T) { + for err, want := range map[error]string{ + fmt.Errorf("reserve: %w", store.ErrHostedEnvironmentFailed): `{"error":{"message":"the hosted environment failed to provision","type":"conflict_error","code":"conflict_error","param":null}}`, + store.ErrEnvironmentUnavailable: `{"error":{"message":"The environment is no longer available for new input.","type":"conflict_error","code":"environment_unavailable","param":null}}`, + } { + response := httptest.NewRecorder() + writeStoreError(response, httptest.NewRequest(http.MethodPost, "/v1/agents/sessions/session/events", nil), err) + body, _ := io.ReadAll(response.Body) + if response.Code != http.StatusConflict || strings.TrimSpace(string(body)) != want { + t.Fatal(response.Code, string(body)) + } + } + if !errors.Is(store.ErrHostedEnvironmentFailed, store.ErrEnvironmentUnavailable) { + t.Fatal("internal callers no longer see an unavailable Environment") + } +} diff --git a/services/agents-api/internal/api/inputs.go b/services/agents-api/internal/api/inputs.go index 25e2f82f2..aeed6e20d 100644 --- a/services/agents-api/internal/api/inputs.go +++ b/services/agents-api/internal/api/inputs.go @@ -24,7 +24,7 @@ type Option func(*Handler) func WithExecution(s InputSubmitter) Option { return func(h *Handler) { h.inputs = s } } // @Summary Submit Session input events -// @Description An empty events array is a resource-authorized no-op; it creates no execution retry identity, Turn, Item or input receipt. For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted profile accepts text-only messages; qualified Codex and Claude SDK openai_hosted profiles also accept inline PNG/JPEG. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 202 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 202 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors; exact hosted failure mapping is unverified. Input the Session cannot accept in its current state, such as a result after cancellation or a batch while earlier input is pending, and a result that differs from the call's saved result return 409 with type and code conflict_error; reusing an Idempotency-Key with a different batch returns the local 409 idempotency_conflict. Inside an owned Session, a result for an unknown call or for a call of another Turn returns 400 invalid_request_error and changes nothing; missing and foreign Sessions return 404. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK accepts text results and, on none and qualified openai_hosted, successful inline PNG/JPEG results, preserving ordered content; error images and remote references reject before admission. Native image resizing may change bytes. Runtime image-result support is checked only for image-bearing delivery. Codex and Claude SDK on none and qualified openai_hosted accept ordered inline PNG/JPEG image messages. Self-hosted profiles and other engines remain text-only; remote image URLs are unsupported. Image references are retained unchanged without service-side downloads. +// @Description An empty events array is a resource-authorized no-op; it creates no execution retry identity, Turn, Item or input receipt. For environment none, atomically accepts text messages, cancellation and function results. Messages steer active work or start a queued Turn. The supported self_hosted profile accepts text-only messages; qualified Codex and Claude SDK openai_hosted profiles also accept inline PNG/JPEG. Under the Session lock, matching retries retain their original target; new active messages append to the current Turn, while idle messages reserve work and wait up to the original five-minute connection/admission deadline. Return 202 only after durable admission, without claiming native application; active messages create no Turn or reservation. Cancellation-only prepared-environment batches use existing durable cancellation admission and return 202 without waiting for native exit; a new cancellation conflicts while a pre-Turn reservation is pending. Homogeneous tool_result-only prepared-environment batches reuse existing scoped result admission and application receipts without creating a Turn or bypassing a pending reservation. Mixed prepared-environment batches remain unsupported. HTTP expiry/cancellation use local 409 environment_input_expired/environment_input_cancelled errors. New input on a Session whose hosted Environment failed to provision returns the observed 409 conflict_error "the hosted environment failed to provision"; input already waiting when it fails and expired Environments keep the local 409 environment_unavailable. Input the Session cannot accept in its current state, such as a result after cancellation or a batch while earlier input is pending, and a result that differs from the call's saved result return 409 with type and code conflict_error; reusing an Idempotency-Key with a different batch returns the local 409 idempotency_conflict. Inside an owned Session, a result for an unknown call or for a call of another Turn returns 400 invalid_request_error and changes nothing; missing and foreign Sessions return 404. Losing execution ownership returns 503. The response write deadline accommodates the admission window for either prepared Environment, independently of new-hosted-admission and executor URL settings. Disconnecting the waiting HTTP request does not cancel retained work or restart its deadline. Retry keys identify the whole ordered batch. Function output accepts text or ordered text/image parts subject to engine support; Claude SDK accepts text results and, on none and qualified openai_hosted, successful inline PNG/JPEG results, preserving ordered content; error images and remote references reject before admission. Native image resizing may change bytes. Runtime image-result support is checked only for image-bearing delivery. Codex and Claude SDK on none and qualified openai_hosted accept ordered inline PNG/JPEG image messages. Self-hosted profiles and other engines remain text-only; remote image URLs are unsupported. Image references are retained unchanged without service-side downloads. // @Tags Sessions // @Accept json // @Security BearerAuth diff --git a/services/agents-api/internal/api/session_response.go b/services/agents-api/internal/api/session_response.go index dd9da8f89..c13c1b98a 100644 --- a/services/agents-api/internal/api/session_response.go +++ b/services/agents-api/internal/api/session_response.go @@ -93,6 +93,16 @@ func sessionResponse(session store.Session, executorURL string) (v1.Session, err response.RequiredActions = append(response.RequiredActions, v1.RequiredAction{Type: "environment_connection", EnvironmentID: activity.EnvironmentID}) } } + // A hosted provisioning failure is terminal and supersedes the settled input + // activity: the Session reports its safe reason and failure time. + if failure := session.EnvironmentFailure; failure != nil { + if cfg.Environment.Type != "openai_hosted" { + return v1.Session{}, errors.New("unsupported stored environment failure") + } + reason := failure.Reason + response.Status, response.Error, response.LastActiveAt = "failed", &reason, failure.FailedAt.Unix() + response.RequiredActions = []v1.RequiredAction{} + } return response, nil } diff --git a/services/agents-api/internal/api/stream.go b/services/agents-api/internal/api/stream.go index 532d6af15..aeda62815 100644 --- a/services/agents-api/internal/api/stream.go +++ b/services/agents-api/internal/api/stream.go @@ -21,7 +21,7 @@ type eventStore interface { } // @Summary Stream live Session events -// @Description Live-only events, including command output fragments from capable Codex peers as agent.output.command_execution_output.delta with stable Item/output indexes. Native text conversion and output quotas apply; completion snapshots remain authoritative. Reconnect through Session, Turn and Items reads; missed events are not replayed. A lagging stream closes with an error when its bounded buffer is exceeded. Session activity includes immutable pending-input connection actions before Turn creation; self_hosted environments use the same safe output as Session retrieval. +// @Description Live-only events, including command output fragments from capable Codex peers as agent.output.command_execution_output.delta with stable Item/output indexes. Native text conversion and output quotas apply; completion snapshots remain authoritative. Reconnect through Session, Turn and Items reads; missed events are not replayed. A lagging stream closes with an error when its bounded buffer is exceeded. When a hosted Environment fails to provision, the stream sends agent.session.environment.failed, an error event (environment_error/sandbox_error with the safe step and exit-status reason, never command output) and agent.session.failed, then ends. Session activity includes immutable pending-input connection actions before Turn creation; self_hosted environments use the same safe output as Session retrieval. // @Tags Events // @Produce text/event-stream // @Security BearerAuth @@ -111,7 +111,7 @@ func (h *Handler) serveSessionEvents(w http.ResponseWriter, r *http.Request, eve return } cursor = change.Sequence - if settlement != nil && settlingEvent(change) { + if terminalEvent(change) || settlement != nil && settlingEvent(change) { return } recheck = recheck || sessionStatusEvent(change.Event.Type) @@ -197,6 +197,13 @@ func settlingEvent(change store.SessionChange) bool { return false } +// terminalEvent reports the agent.session.failed of a hosted provisioning +// failure. The Session can never run again, so GET streams end after it too, as +// officially observed; other failures leave GET streams open. +func terminalEvent(change store.SessionChange) bool { + return change.Event.Type == "agent.session.failed" && change.EnvironmentFailure != nil +} + func sessionStatusEvent(eventType string) bool { switch eventType { case "agent.session.in_progress", "agent.session.requires_action", "agent.session.idle", "agent.session.failed": @@ -207,7 +214,7 @@ func sessionStatusEvent(eventType string) bool { func streamResponse(session store.Session, change store.SessionChange, executorURL string) (v1.SessionEvent, error) { event := change.Event - if change.Turn == nil && change.EnvironmentInputActivity == nil { + if change.Turn == nil && change.EnvironmentInputActivity == nil && change.EnvironmentFailure == nil { return withTurnUsage(event), nil } if change.Turn != nil && strings.HasPrefix(event.Type, "agent.session.turn.") { @@ -218,7 +225,7 @@ func streamResponse(session store.Session, change store.SessionChange, executorU event.SessionID = "" session.RequiredActions = change.RequiredActions session.LastTurn, session.Usage = change.Turn, change.SessionUsage - session.EnvironmentInputActivity = change.EnvironmentInputActivity + session.EnvironmentInputActivity, session.EnvironmentFailure = change.EnvironmentInputActivity, change.EnvironmentFailure value, err := sessionResponse(session, executorURL) event.Session = &value return event, err diff --git a/services/agents-api/internal/db/queries/environment_connections.sql b/services/agents-api/internal/db/queries/environment_connections.sql index 4c2dc9dab..044a8ec8c 100644 --- a/services/agents-api/internal/db/queries/environment_connections.sql +++ b/services/agents-api/internal/db/queries/environment_connections.sql @@ -12,6 +12,11 @@ UPDATE environment_connections SET revision = $2 WHERE environment_id = $1; -- name: SetEnvironmentConnectionStatus :exec UPDATE environments SET status = $2 WHERE id = $1; +-- name: RecordEnvironmentFailure :one +UPDATE environments SET status = 'failed', failure_reason = $2, failed_at = clock_timestamp() +WHERE id = $1 AND status NOT IN ('failed', 'expired') +RETURNING failed_at; + -- name: DeleteEnvironmentConnection :exec DELETE FROM environment_connections WHERE environment_id = $1; diff --git a/services/agents-api/internal/db/sqlc/environment_connections.sql.go b/services/agents-api/internal/db/sqlc/environment_connections.sql.go index 7d850cf27..e9c31ad62 100644 --- a/services/agents-api/internal/db/sqlc/environment_connections.sql.go +++ b/services/agents-api/internal/db/sqlc/environment_connections.sql.go @@ -81,6 +81,24 @@ func (q *Queries) ListEnvironmentConnections(ctx context.Context, id pgtype.UUID return items, nil } +const recordEnvironmentFailure = `-- name: RecordEnvironmentFailure :one +UPDATE environments SET status = 'failed', failure_reason = $2, failed_at = clock_timestamp() +WHERE id = $1 AND status NOT IN ('failed', 'expired') +RETURNING failed_at +` + +type RecordEnvironmentFailureParams struct { + ID pgtype.UUID `json:"id"` + FailureReason pgtype.Text `json:"failure_reason"` +} + +func (q *Queries) RecordEnvironmentFailure(ctx context.Context, arg RecordEnvironmentFailureParams) (pgtype.Timestamptz, error) { + row := q.db.QueryRow(ctx, recordEnvironmentFailure, arg.ID, arg.FailureReason) + var failed_at pgtype.Timestamptz + err := row.Scan(&failed_at) + return failed_at, err +} + const replaceEnvironmentConnection = `-- name: ReplaceEnvironmentConnection :exec INSERT INTO environment_connections (environment_id, generation, revision) VALUES ($1, $2, 0) diff --git a/services/agents-api/internal/db/sqlc/environments.sql.go b/services/agents-api/internal/db/sqlc/environments.sql.go index db51912fc..986ad4933 100644 --- a/services/agents-api/internal/db/sqlc/environments.sql.go +++ b/services/agents-api/internal/db/sqlc/environments.sql.go @@ -26,7 +26,7 @@ func (q *Queries) CreateEnvironment(ctx context.Context, arg CreateEnvironmentPa } const getEnvironment = `-- name: GetEnvironment :one -SELECT e.id, e.session_id, e.status, e.created_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL ` @@ -50,6 +50,8 @@ func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams) &i.Environment.SessionID, &i.Environment.Status, &i.Environment.CreatedAt, + &i.Environment.FailureReason, + &i.Environment.FailedAt, &i.TenantID, &i.Configuration, ) @@ -57,7 +59,7 @@ func (q *Queries) GetEnvironment(ctx context.Context, arg GetEnvironmentParams) } const getSessionEnvironment = `-- name: GetSessionEnvironment :one -SELECT e.id, e.session_id, e.status, e.created_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration +SELECT e.id, e.session_id, e.status, e.created_at, e.failure_reason, e.failed_at, s.tenant_id, (s.configuration->'environment')::jsonb AS configuration FROM environments e JOIN sessions s ON s.id = e.session_id WHERE s.tenant_id = $1 AND s.id = $2 AND s.deleted_at IS NULL ` @@ -81,6 +83,8 @@ func (q *Queries) GetSessionEnvironment(ctx context.Context, arg GetSessionEnvir &i.Environment.SessionID, &i.Environment.Status, &i.Environment.CreatedAt, + &i.Environment.FailureReason, + &i.Environment.FailedAt, &i.TenantID, &i.Configuration, ) diff --git a/services/agents-api/internal/db/sqlc/models.go b/services/agents-api/internal/db/sqlc/models.go index 4cfc15f88..94afb91b0 100644 --- a/services/agents-api/internal/db/sqlc/models.go +++ b/services/agents-api/internal/db/sqlc/models.go @@ -35,10 +35,12 @@ type Device struct { } type Environment struct { - ID pgtype.UUID `json:"id"` - SessionID pgtype.UUID `json:"session_id"` - Status string `json:"status"` - CreatedAt pgtype.Timestamptz `json:"created_at"` + ID pgtype.UUID `json:"id"` + SessionID pgtype.UUID `json:"session_id"` + Status string `json:"status"` + CreatedAt pgtype.Timestamptz `json:"created_at"` + FailureReason pgtype.Text `json:"failure_reason"` + FailedAt pgtype.Timestamptz `json:"failed_at"` } type EnvironmentConnection struct { diff --git a/services/agents-api/internal/execution/runtime_initialization.go b/services/agents-api/internal/execution/runtime_initialization.go index e90212e65..8c709b67a 100644 --- a/services/agents-api/internal/execution/runtime_initialization.go +++ b/services/agents-api/internal/execution/runtime_initialization.go @@ -97,6 +97,7 @@ func (r *runtimeLifecycle) advanceInitialization(ctx context.Context) error { r.initializing = nil return err } + step := store.ProvisioningFailure{Step: store.ProvisioningInitialFile} if active.next < active.files { var file store.InitialFileMetadata var body []byte @@ -105,12 +106,25 @@ func (r *runtimeLifecycle) advanceInitialization(ctx context.Context) error { err = installInitialFile(operation, r.config.Provider, runtimeReference(owner), file, body) } } else { - err = runRuntimeSetup(operation, r.config.Provider, runtimeReference(owner), active.operations[active.next-active.files]) + setup := active.operations[active.next-active.files] + step = setup.provisioningFailure(0) + err = runRuntimeSetup(operation, r.config.Provider, runtimeReference(owner), setup) } if err != nil { // Clearing the in-memory owner makes the next observation request cleanup, // even when the failed operation consumed its entire deadline. r.initializing = nil + var failed *runtimeStepFailure + if errors.As(err, &failed) { + // A confirmed failed step records its safe reason now. Unknown effects + // keep the generic reason recorded by that later cleanup. + step.ExitCode = failed.exitCode + record, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, cleanupErr := r.store.FailRuntimeInitialization(record, owner, step); cleanupErr != nil { + return errors.Join(err, cleanupErr) + } + } return err } active.next++ @@ -139,10 +153,16 @@ func installInitialFile(ctx context.Context, provider sandbox.Provider, referenc Outcome string `json:"outcome"` SizeBytes *int64 `json:"size_bytes"` } - if result.ExitCode != 0 || result.Stderr != "" || json.Unmarshal([]byte(result.Stdout), &receipt) != nil || receipt.Version != 1 || receipt.Outcome != "completed" || receipt.SizeBytes == nil || *receipt.SizeBytes != int64(len(body)) { - return errors.New("initial environment file installation unconfirmed") + valid := result.Stderr == "" && json.Unmarshal([]byte(result.Stdout), &receipt) == nil && receipt.Version == 1 + if valid && result.ExitCode == 0 && receipt.Outcome == "completed" && receipt.SizeBytes != nil && *receipt.SizeBytes == int64(len(body)) { + return nil + } + if valid && receipt.Outcome == "failed" { + // The writer exits 0 with a failed receipt when it committed nothing; + // "unknown" and every other result stay generic. + return &runtimeStepFailure{} } - return nil + return errors.New("initial environment file installation unconfirmed") } // Isolated Python creates only fd-anchored workspace parents, then replaces itself with the existing atomic writer. diff --git a/services/agents-api/internal/execution/runtime_setup.go b/services/agents-api/internal/execution/runtime_setup.go index 8b839fbb8..cecd74def 100644 --- a/services/agents-api/internal/execution/runtime_setup.go +++ b/services/agents-api/internal/execution/runtime_setup.go @@ -26,6 +26,24 @@ type runtimeSetupOperation struct { Packages []string `json:"packages,omitempty"` Command string `json:"command,omitempty"` CWD string `json:"cwd,omitempty"` + // Index is the setup command position, used only for the public failure label. + Index int `json:"-"` +} + +// runtimeStepFailure is a confirmed failed initialization receipt. It holds only +// the Runtime-reported exit status (0 when absent), never command or output. +type runtimeStepFailure struct{ exitCode int } + +func (*runtimeStepFailure) Error() string { return "environment initialization operation failed" } + +// provisioningFailure labels a confirmed failed setup operation for the Store, +// which composes the public reason. Other actions keep the generic reason. +func (operation runtimeSetupOperation) provisioningFailure(exitCode int) store.ProvisioningFailure { + switch operation.Action { + case "setup", "python", "npm", "system", "skill": + return store.ProvisioningFailure{Step: operation.Action, Index: operation.Index, ExitCode: exitCode} + } + return store.ProvisioningFailure{} } func setupOperations(setup store.EnvironmentSetup) []runtimeSetupOperation { @@ -55,12 +73,12 @@ func setupOperations(setup store.EnvironmentSetup) []runtimeSetupOperation { if len(setup.Packages.Python) > 0 { result = append(result, runtimeSetupOperation{Version: 1, Action: "python", Network: network, Packages: setup.Packages.Python}) } - for _, command := range setup.Commands { + for i, command := range setup.Commands { cwd := command.CWD if cwd == "" { cwd = "/workspace" } - result = append(result, runtimeSetupOperation{Version: 1, Action: "setup", Network: network, Command: command.Command, CWD: cwd}) + result = append(result, runtimeSetupOperation{Version: 1, Action: "setup", Network: network, Command: command.Command, CWD: cwd, Index: i}) } if len(setup.Skills)+len(setup.Plugins)+len(setup.CapabilityDirectories) > 0 { sources := agentcapabilities.Input{Plugins: setup.PluginMetadata(), Directories: setup.CapabilityDirectories} @@ -98,11 +116,22 @@ func runRuntimeSetup(ctx context.Context, provider sandbox.Provider, reference s return err } var receipt struct { - Version int `json:"version"` - Outcome string `json:"outcome"` + Version int `json:"version"` + Outcome string `json:"outcome"` + ExitCode *int `json:"exit_code"` } - if result.ExitCode != 0 || result.Stderr != "" || json.Unmarshal([]byte(result.Stdout), &receipt) != nil || receipt.Version != 1 || receipt.Outcome != "completed" { - return errors.New("environment initialization operation unconfirmed") + valid := result.Stderr == "" && json.Unmarshal([]byte(result.Stdout), &receipt) == nil && receipt.Version == 1 + if valid && result.ExitCode == 0 && receipt.Outcome == "completed" { + return nil + } + // The initializer confirms a failed step with status 1 and, for a sandboxed + // step, that step's exit status only. Images without exit_code stay generic. + if valid && result.ExitCode == 1 && receipt.Outcome == "failed" && operation.Capabilities == nil { + failure := &runtimeStepFailure{} + if receipt.ExitCode != nil && *receipt.ExitCode > 0 && *receipt.ExitCode < 256 { + failure.exitCode = *receipt.ExitCode + } + return failure } - return nil + return errors.New("environment initialization operation unconfirmed") } diff --git a/services/agents-api/internal/store/environment_connections.go b/services/agents-api/internal/store/environment_connections.go index 59b14d744..54685242f 100644 --- a/services/agents-api/internal/store/environment_connections.go +++ b/services/agents-api/internal/store/environment_connections.go @@ -102,21 +102,29 @@ func (s *Store) withEnvironmentConnection(ctx context.Context, tenant, environme } func recordEnvironmentConnection(ctx context.Context, q *sqlc.Queries, row sqlc.GetSessionEnvironmentRow, status string) error { - var config struct { - Type string `json:"type"` - } - if err := json.Unmarshal(row.Configuration, &config); err != nil || (config.Type != "self_hosted" && config.Type != "openai_hosted") { - return errors.New("invalid stored Environment type") + if _, err := storedEnvironmentType(row); err != nil { + return err } - if status != "connected" && status != "disconnected" && status != "failed" { + if status != "connected" && status != "disconnected" { return ErrInvalidInput } if err := q.SetEnvironmentConnectionStatus(ctx, sqlc.SetEnvironmentConnectionStatusParams{ID: row.Environment.ID, Status: status}); err != nil { return err } - state := &v1.SessionEnvironmentState{ID: uuid.UUID(row.Environment.ID.Bytes).String(), Type: config.Type, Status: status} + return recordEnvironmentState(ctx, q, row, status) +} + +// recordEnvironmentState appends the pinned Environment state event for an +// already committed status. A failure uses the observed official error; the +// failed step travels only in the separate error event and Session error. +func recordEnvironmentState(ctx context.Context, q *sqlc.Queries, row sqlc.GetSessionEnvironmentRow, status string) error { + kind, err := storedEnvironmentType(row) + if err != nil { + return err + } + state := &v1.SessionEnvironmentState{ID: uuid.UUID(row.Environment.ID.Bytes).String(), Type: kind, Status: status} if status == "failed" { - state.Error = &v1.StreamError{Code: "environment_unavailable", Type: "server_error", Message: "The environment could not be prepared for execution."} + state.Error = &v1.StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."} } return recordSessionChange(ctx, q, row.Environment.SessionID, SessionChange{Event: v1.SessionEvent{ Type: "agent.session.environment." + status, @@ -124,6 +132,16 @@ func recordEnvironmentConnection(ctx context.Context, q *sqlc.Queries, row sqlc. }}) } +func storedEnvironmentType(row sqlc.GetSessionEnvironmentRow) (string, error) { + var config struct { + Type string `json:"type"` + } + if err := json.Unmarshal(row.Configuration, &config); err != nil || (config.Type != "self_hosted" && config.Type != "openai_hosted") { + return "", errors.New("invalid stored Environment type") + } + return config.Type, nil +} + func parseConnectionGeneration(value string) (pgtype.UUID, error) { id, err := parseID(value) if err != nil || id.Bytes == [16]byte{} { diff --git a/services/agents-api/internal/store/environment_inputs.go b/services/agents-api/internal/store/environment_inputs.go index 28966e1d1..027a790e5 100644 --- a/services/agents-api/internal/store/environment_inputs.go +++ b/services/agents-api/internal/store/environment_inputs.go @@ -26,6 +26,11 @@ const ( // still waits for admission. It remains a Turn conflict for internal callers. var ErrSessionInputPending = fmt.Errorf("%w: session input is still pending", ErrTurnConflict) +// ErrHostedEnvironmentFailed rejects new input on a Session whose hosted +// Environment failed to provision. It remains ErrEnvironmentUnavailable for +// internal callers; an expired Environment keeps that plain error. +var ErrHostedEnvironmentFailed = fmt.Errorf("%w: the hosted environment failed to provision", ErrEnvironmentUnavailable) + // EnvironmentInputReservation is private admission state, not a public Session projection. type EnvironmentInputReservation struct { ID string @@ -86,6 +91,11 @@ func (s *Store) ReserveEnvironmentInput(ctx context.Context, tenantID, sessionID } else if err != nil { return err } + if environment.Environment.Status == "failed" { + if kind, err := storedEnvironmentType(environment); err == nil && kind == "openai_hosted" { + return ErrHostedEnvironmentFailed + } + } if environment.Environment.Status == "failed" || environment.Environment.Status == "expired" { return ErrEnvironmentUnavailable } diff --git a/services/agents-api/internal/store/environments_migration_test.go b/services/agents-api/internal/store/environments_migration_test.go index c9cdc21a1..2566751b2 100644 --- a/services/agents-api/internal/store/environments_migration_test.go +++ b/services/agents-api/internal/store/environments_migration_test.go @@ -78,7 +78,6 @@ func TestEnvironmentMigrationPreservesHistoryAndGuardsIdentity(t *testing.T) { t.Fatal(err) } t.Cleanup(migrated.Close) - s := New(migrated) sessionID, environmentID := uuid.NewString(), uuid.NewString() tx, err := migrated.Begin(ctx) @@ -124,8 +123,11 @@ func TestEnvironmentMigrationPreservesHistoryAndGuardsIdentity(t *testing.T) { if err := <-outcome; err == nil || !strings.Contains(err.Error(), "Cannot remove durable Environment identities") { t.Fatal("concurrent downgrade discarded identity", err) } - retained, err := s.GetEnvironment(ctx, tenant, environmentID) - if err != nil || retained.ID != environmentID || retained.SessionID != sessionID { + // Read the version-20 row directly: current Store queries select columns + // that later migrations add to environments. + var retained string + if err := migrated.QueryRow(ctx, `SELECT e.session_id::text FROM environments e JOIN sessions s ON s.id = e.session_id + WHERE s.tenant_id = $1 AND e.id = $2 AND s.deleted_at IS NULL`, tenant, environmentID).Scan(&retained); err != nil || retained != sessionID { t.Fatal("downgrade destroyed ownership", retained, err) } } diff --git a/services/agents-api/internal/store/hosted_initialization_failure_public_test.go b/services/agents-api/internal/store/hosted_initialization_failure_public_test.go new file mode 100644 index 000000000..ff5fe8164 --- /dev/null +++ b/services/agents-api/internal/store/hosted_initialization_failure_public_test.go @@ -0,0 +1,384 @@ +package store_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "testing" + "time" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/api" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/credentialcrypto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +const hostedFailureCanary = "CANARY-hosted-init-7c21" + +// hostedFailureProvider fails one initialization step with a controlled result. +// Every failure it reports is produced next to canary output, which the +// initializer discards and Core must never publish. +type hostedFailureProvider struct { + lifecycleProvider + fail string // runtime-initialize action, or "file" for the initial file writer + skip int // matching steps that succeed before the failure + result sandbox.CommandResult + err error + steps []string +} + +func (p *hostedFailureProvider) RunCommand(_ context.Context, _ sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + action := "file" + if c.Args[len(c.Args)-1] == "/usr/local/bin/agents-api-runtime-initialize" { + var operation struct { + Action string `json:"action"` + } + if json.Unmarshal(c.Stdin, &operation) != nil || operation.Action == "" { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + action = operation.Action + } + p.mu.Lock() + p.steps = append(p.steps, action) + p.mu.Unlock() + if action == p.fail { + if p.skip == 0 { + return p.result, p.err + } + p.skip-- + } + if action == "file" { + size, err := strconv.Atoi(c.Args[len(c.Args)-1]) + if err != nil { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + return sandbox.CommandResult{Stdout: fmt.Sprintf(`{"version":1,"outcome":"completed","size_bytes":%d}`, size)}, nil + } + return sandbox.CommandResult{Stdout: `{"version":1,"outcome":"completed"}`}, nil +} + +func hostedFailureStore(t *testing.T) *store.Store { + t.Helper() + _, pool := store.NewManagedTestStore(t) + cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) + if err != nil { + t.Fatal(err) + } + return store.NewWithCredentialCipher(pool, cipher) +} + +func hostedFailureSession(t *testing.T, s *store.Store, tenant string, input store.CreateSessionInput) (store.Session, store.Environment) { + t.Helper() + input.Creator, input.Engine, input.IdempotencyKey = store.FixtureCreator(), "codex", uuid.NewString() + input.Configuration = json.RawMessage(`{"agent":{"id":"agent_test","model":"test-model","tools":[]},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`) + if input.Initialization.Env == nil { + input.Initialization.Env = map[string]string{"SCAN_VALUE": hostedFailureCanary} + } + session, err := s.CreateSession(t.Context(), tenant, input) + if err != nil { + t.Fatal(err) + } + environment, err := s.GetSessionEnvironment(t.Context(), tenant, session.ID) + if err != nil { + t.Fatal(err) + } + return session, environment +} + +func failHostedInitialization(t *testing.T, s *store.Store, tenant string, environment store.Environment, p *hostedFailureProvider) { + t.Helper() + key := uuid.NewString() + w, _ := managedWorker(t, s, key, p) + if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { + t.Fatal(err) + } + reconcileManagedState(t, w, s, tenant, environment.ID, "released") +} + +// H1/H2/H3/H4: one transaction records the Environment failure, an error event +// with the safe reason and agent.session.failed; reads and events agree, and a +// confirmed step names only its label and exit status. +func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { + failed := func(stdout string) sandbox.CommandResult { return sandbox.CommandResult{ExitCode: 1, Stdout: stdout} } + commands := []store.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 0"}, {Command: "echo " + hostedFailureCanary + "; exit 3"}, {Command: "touch never"}} + type failure struct { + fail string + skip int + result sandbox.CommandResult + err error + } + for _, test := range []struct { + name string + input store.CreateSessionInput + p failure + reason string + steps []string + }{ + {"setup exit status", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, + failure{fail: "setup", result: failed(`{"version":1,"outcome":"failed","exit_code":3}`)}, + `Failed to provision environment: script "setup_commands[0]" failed with exit code 3`, []string{"configure", "setup"}}, + {"later setup command", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands}}, + failure{fail: "setup", skip: 1, result: failed(`{"version":1,"outcome":"failed","exit_code":3}` + "\n")}, + `Failed to provision environment: script "setup_commands[1]" failed with exit code 3`, []string{"configure", "setup", "setup"}}, + {"python package", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Packages: v1.EnvironmentPackages{Python: []string{"parsar-nonexistent-zz"}}, Commands: commands[2:]}}, + failure{fail: "python", result: failed(`{"version":1,"outcome":"failed","exit_code":1}`)}, + `Failed to provision environment: script "Python package installation" failed with exit code 1`, []string{"configure", "python"}}, + {"old image without exit status", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, + failure{fail: "setup", result: failed(`{"version":1,"outcome":"failed"}`)}, + "Failed to provision environment: initialization did not complete", []string{"configure", "setup"}}, + {"unknown effect", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, + failure{fail: "setup", err: sandbox.ErrCommandUnconfirmed}, + "Failed to provision environment: initialization did not complete", []string{"configure", "setup"}}, + {"output instead of a receipt", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, + failure{fail: "setup", result: sandbox.CommandResult{ExitCode: 3, Stdout: hostedFailureCanary, Stderr: hostedFailureCanary}}, + "Failed to provision environment: initialization did not complete", []string{"configure", "setup"}}, + {"initial file", store.CreateSessionInput{InitialFiles: []store.InitialFile{{Type: "inline", Path: "/workspace/a", Data: []byte(hostedFailureCanary)}}}, + failure{fail: "file", result: sandbox.CommandResult{Stdout: `{"version":1,"outcome":"failed","error":"write_failed"}`}}, + "Failed to provision environment: initial file installation failed", []string{"file"}}, + } { + t.Run(test.name, func(t *testing.T) { + s := hostedFailureStore(t) + tenant := uuid.NewString() + session, environment := hostedFailureSession(t, s, tenant, test.input) + p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, + fail: test.p.fail, skip: test.p.skip, result: test.p.result, err: test.p.err} + failHostedInitialization(t, s, tenant, environment, p) + if !reflect.DeepEqual(p.steps, test.steps) || p.kills != 1 { + t.Fatal("failed initialization continued or was not reclaimed", p.steps, p.kills) + } + + read, err := s.GetSession(t.Context(), tenant, session.ID) + if err != nil || read.Environment.Status != "failed" || read.EnvironmentFailure == nil || read.EnvironmentFailure.Reason != test.reason || read.EnvironmentInputActivity != nil || read.LastTurn != nil { + t.Fatal("Session read", read.EnvironmentFailure, err) + } + page, err := s.ListSessions(t.Context(), tenant, "", 10, false, nil) + if err != nil || len(page.Sessions) != 1 || !reflect.DeepEqual(page.Sessions[0].EnvironmentFailure, read.EnvironmentFailure) { + t.Fatal("Session list", page, err) + } + events, err := s.ListSessionEvents(t.Context(), tenant, session.ID, 0) + if err != nil || len(events) != 3 { + t.Fatal("failure events", events, err) + } + if event := events[0].Event; event.Type != "agent.session.environment.failed" || event.Environment == nil || event.Environment.ID != environment.ID || + event.Environment.Status != "failed" || event.Environment.Type != "openai_hosted" || + !reflect.DeepEqual(event.Environment.Error, &v1.StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."}) { + t.Fatal("environment failure event", event) + } + if event := events[1].Event; event.Type != "error" || !reflect.DeepEqual(event.Error, &v1.StreamError{Type: "environment_error", Code: "sandbox_error", Message: test.reason}) { + t.Fatal("error event", event) + } + last := events[2] + if last.Event.Type != "agent.session.failed" || last.EnvironmentFailure == nil || last.EnvironmentFailure.Reason != test.reason || + !last.EnvironmentFailure.FailedAt.Equal(read.EnvironmentFailure.FailedAt) || last.EnvironmentInputActivity != nil || !last.Settled { + t.Fatal("failed snapshot", last) + } + if _, err := s.ReserveEnvironmentInput(t.Context(), tenant, session.ID, "later", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"later"}`)}}); !errors.Is(err, store.ErrHostedEnvironmentFailed) { + t.Fatal("failed hosted Environment admitted input", err) + } + raw, _ := json.Marshal(events) + if strings.Contains(string(raw), hostedFailureCanary) || strings.Contains(test.reason, hostedFailureCanary) { + t.Fatal("initialization output reached public events") + } + // Tenant B cannot observe the failure. + other := uuid.NewString() + if _, err := s.GetSession(t.Context(), other, session.ID); !errors.Is(err, store.ErrNotFound) { + t.Fatal("foreign Session read", err) + } + if _, err := s.ListSessionEvents(t.Context(), other, session.ID, 0); !errors.Is(err, store.ErrNotFound) { + t.Fatal("foreign Session events", err) + } + if page, err := s.ListSessions(t.Context(), other, "", 10, false, nil); err != nil || len(page.Sessions) != 0 { + t.Fatal("foreign Session list", page, err) + } + }) + } +} + +// A pending initial input settles exactly as before; the one failed snapshot +// carries both that settlement and the provisioning failure. +func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { + s := hostedFailureStore(t) + tenant := uuid.NewString() + session, environment := hostedFailureSession(t, s, tenant, store.CreateSessionInput{ + Initialization: store.EnvironmentSetup{Commands: []store.SetupCommand{{Command: "exit 3"}}}, + InitialInputs: []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"initial"}`)}}, + }) + p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", + result: sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}} + failHostedInitialization(t, s, tenant, environment, p) + read, err := s.GetSession(t.Context(), tenant, session.ID) + if err != nil || read.PendingInput || read.EnvironmentInputActivity == nil || read.EnvironmentInputActivity.Status != "failed" || + read.EnvironmentInputActivity.Failure != "environment_unavailable" || read.EnvironmentFailure == nil { + t.Fatal("pending input settlement", read.EnvironmentInputActivity, read.EnvironmentFailure, err) + } + events, err := s.ListSessionEvents(t.Context(), tenant, session.ID, 0) + if err != nil { + t.Fatal(err) + } + var types []string + for _, event := range events { + types = append(types, event.Event.Type) + } + if !reflect.DeepEqual(types[len(types)-3:], []string{"agent.session.environment.failed", "error", "agent.session.failed"}) || strings.Count(strings.Join(types, " "), "agent.session.failed") != 1 { + t.Fatal("failure events", types) + } + if last := events[len(events)-1]; last.EnvironmentInputActivity == nil || last.EnvironmentInputActivity.Status != "failed" || last.EnvironmentFailure == nil { + t.Fatal("failed snapshot", last) + } +} + +// H1/H5/H6/H7 over HTTP: retrieve, list and the live stream agree; the GET +// stream ends after agent.session.failed; later input gets the observed 409; +// delete succeeds; tenant B sees nothing; the canary never appears. +func TestHostedInitializationFailurePublicHTTP(t *testing.T) { + s := hostedFailureStore(t) + tenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() + session, environment := hostedFailureSession(t, s, tenant, store.CreateSessionInput{ + Initialization: store.EnvironmentSetup{Commands: []store.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 3"}}}, + Metadata: map[string]string{"case": "setup-exit3"}, + }) + key := uuid.NewString() + p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", + result: sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}} + w, _ := managedWorker(t, s, key, p) + auth, err := api.NewAuthenticator([]api.APIKey{ + {OrganizationID: "test-org", ProjectID: tenant, SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: device.HashCredential(token), TenantID: tenant}, + {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "tenant-b", TokenSHA256: device.HashCredential(foreign), TenantID: uuid.NewString()}, + }) + if err != nil { + t.Fatal(err) + } + handler, err := api.NewHandler(s, auth, "codex", api.WithExecution(w)) + if err != nil { + t.Fatal(err) + } + server := httptest.NewServer(handler) + defer server.Close() + var bodies []string + call := func(credential, method, path, body string) (int, map[string]any) { + t.Helper() + request, err := http.NewRequest(method, server.URL+path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", "Bearer "+credential) + request.Header.Set("OpenAI-Beta", "agents=v1") + if body != "" { + request.Header.Set("Content-Type", "application/json") + } + response, err := server.Client().Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + raw, _ := io.ReadAll(response.Body) + bodies = append(bodies, string(raw)) + var value map[string]any + _ = json.Unmarshal(raw, &value) + return response.StatusCode, value + } + + live := openStream(t, server, token, http.MethodGet, "/v1/agents/sessions/"+session.ID+"/events", "", "") + defer live.stop() + if line := live.next(t); line != ": connected" { + t.Fatal(line) + } + if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { + t.Fatal(err) + } + reconcileManagedState(t, w, s, tenant, environment.ID, "released") + + reason := `Failed to provision environment: script "setup_commands[0]" failed with exit code 3` + read, err := s.GetSession(t.Context(), tenant, session.ID) + if err != nil || read.EnvironmentFailure == nil { + t.Fatal(err) + } + failedAt := float64(read.EnvironmentFailure.FailedAt.Unix()) + checkSession := func(value any) { + t.Helper() + got, _ := value.(map[string]any) + if got["id"] != session.ID || got["status"] != "failed" || got["error"] != reason || got["last_active_at"] != failedAt || + !reflect.DeepEqual(got["required_actions"], []any{}) || got["usage"] != nil { + t.Fatal("failed Session projection", got) + } + } + var frames []map[string]any + var names []string + for len(frames) < 3 { + line := live.next(t) + if name, ok := strings.CutPrefix(line, "event: "); ok { + names = append(names, name) + continue + } + data, ok := strings.CutPrefix(line, "data: ") + var frame map[string]any + if !ok || json.Unmarshal([]byte(data), &frame) != nil { + t.Fatal("invalid frame", line) + } + bodies = append(bodies, data) + frames = append(frames, frame) + } + live.ended(t, 5*time.Second) + if !reflect.DeepEqual(names, []string{"agent.session.environment.failed", "error", "agent.session.failed"}) { + t.Fatal("stream order", names) + } + if !reflect.DeepEqual(frames[0]["environment"], map[string]any{"id": environment.ID, "type": "openai_hosted", "status": "failed", + "error": map[string]any{"type": "environment_error", "code": "environment_connection_failed", "message": "The environment failed to connect."}}) { + t.Fatal("environment.failed frame", frames[0]) + } + if !reflect.DeepEqual(frames[1], map[string]any{"type": "error", "event_id": frames[1]["event_id"], "session_id": session.ID, + "error": map[string]any{"type": "environment_error", "code": "sandbox_error", "message": reason, "param": nil}}) { + t.Fatal("error frame", frames[1]) + } + checkSession(frames[2]["session"]) + + if status, body := call(token, http.MethodGet, "/v1/agents/sessions/"+session.ID, ""); status != http.StatusOK { + t.Fatal(status, body) + } else { + checkSession(body) + } + if status, body := call(token, http.MethodGet, "/v1/agents/sessions", ""); status != http.StatusOK { + t.Fatal(status, body) + } else if data, _ := body["data"].([]any); len(data) != 1 { + t.Fatal("list", body) + } else { + checkSession(data[0]) + } + input := `{"events":[{"type":"agent.session.input.message","input":[{"role":"user","content":[{"type":"input_text","text":"Reply with OK."}]}]}]}` + conflict := map[string]any{"error": map[string]any{"type": "conflict_error", "code": "conflict_error", "message": "the hosted environment failed to provision", "param": nil}} + if status, body := call(token, http.MethodPost, "/v1/agents/sessions/"+session.ID+"/events", input); status != http.StatusConflict || !reflect.DeepEqual(body, conflict) { + t.Fatal("later input", status, body) + } + for _, request := range [][2]string{{http.MethodGet, ""}, {http.MethodGet, "/events"}, {http.MethodPost, "/events"}, {http.MethodDelete, ""}} { + body := "" + if request[0] == http.MethodPost { + body = input + } + if status, got := call(foreign, request[0], "/v1/agents/sessions/"+session.ID+request[1], body); status != http.StatusNotFound { + t.Fatal("tenant B", request, status, got) + } + } + if status, body := call(token, http.MethodDelete, "/v1/agents/sessions/"+session.ID, ""); status != http.StatusOK || + !reflect.DeepEqual(body, map[string]any{"id": session.ID, "object": "agent.session.deleted", "deleted": true}) { + t.Fatal("delete", status, body) + } + if status, _ := call(token, http.MethodGet, "/v1/agents/sessions/"+session.ID, ""); status != http.StatusNotFound { + t.Fatal("deleted Session remained", status) + } + for _, body := range bodies { + if strings.Contains(body, hostedFailureCanary) { + t.Fatal("initialization output reached a response", body) + } + } +} diff --git a/services/agents-api/internal/store/runtime_allocation_state.go b/services/agents-api/internal/store/runtime_allocation_state.go index fe18fb969..067ce66ff 100644 --- a/services/agents-api/internal/store/runtime_allocation_state.go +++ b/services/agents-api/internal/store/runtime_allocation_state.go @@ -38,8 +38,20 @@ func (s *Store) SettleRuntimeCreation(ctx context.Context, owner RuntimeAllocati } // RequestRuntimeCleanup revokes future authority before external reclamation. -// Cancellation requests do not prove existing native work has stopped. +// Cancellation requests do not prove existing native work has stopped. A live, +// unexpired Environment fails with the generic provisioning reason. func (s *Store) RequestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation) (RuntimeAllocation, error) { + return s.requestRuntimeCleanup(ctx, owner, provisioningFailureReason) +} + +// FailRuntimeInitialization is RequestRuntimeCleanup after a confirmed failed +// initialization step: the step's safe reason becomes the Session error. Deleted +// Sessions, expired and already terminal Environments keep their existing outcome. +func (s *Store) FailRuntimeInitialization(ctx context.Context, owner RuntimeAllocation, failure ProvisioningFailure) (RuntimeAllocation, error) { + return s.requestRuntimeCleanup(ctx, owner, failure.reason()) +} + +func (s *Store) requestRuntimeCleanup(ctx context.Context, owner RuntimeAllocation, reason string) (RuntimeAllocation, error) { return s.mutateRuntimeAllocation(ctx, owner, false, func(ctx context.Context, q *sqlc.Queries, row sqlc.RuntimeAllocation) (sqlc.RuntimeAllocation, error) { if row.State == "released" { return row, nil @@ -56,12 +68,7 @@ func (s *Store) RequestRuntimeCleanup(ctx context.Context, owner RuntimeAllocati if current.DeletedAt.Valid { err = cancel() } else { - err = withEnvironmentInputActivity(ctx, q, current.SessionID, func() error { - if err := terminateRuntimeEnvironment(ctx, q, current); err != nil { - return err - } - return cancel() - }) + err = terminateRuntimeEnvironment(ctx, q, current, reason, cancel) } if err != nil { return sqlc.RuntimeAllocation{}, err diff --git a/services/agents-api/internal/store/runtime_environment_terminal.go b/services/agents-api/internal/store/runtime_environment_terminal.go index 69bb327dc..06e5d8827 100644 --- a/services/agents-api/internal/store/runtime_environment_terminal.go +++ b/services/agents-api/internal/store/runtime_environment_terminal.go @@ -2,25 +2,137 @@ package store import ( "context" + "fmt" + "time" + + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/db/sqlc" + "github.com/jackc/pgx/v5/pgtype" ) +// provisioningFailureReason is the safe reason for a hosted Environment that +// failed without a confirmed failed step: timeouts, unknown effects, missing or +// old receipts, bootstrap rejection and Core restart during initialization. +const provisioningFailureReason = "Failed to provision environment: initialization did not complete" + +// Provisioning step kinds for ProvisioningFailure.Step. +const ( + ProvisioningSetupCommand = "setup" + ProvisioningPythonPackages = "python" + ProvisioningNPMPackages = "npm" + ProvisioningSystemPackages = "system" + ProvisioningInitialFile = "file" + ProvisioningSkill = "skill" +) + +// ProvisioningFailure identifies a confirmed failed hosted initialization step. +// It cannot carry Runtime output: Step selects a fixed label, Index is the setup +// command position and ExitCode is the Runtime-reported status (0 when absent). +type ProvisioningFailure struct { + Step string + Index int + ExitCode int +} + +// reason renders the public Session error. The setup_commands and Python package +// labels match observed official errors (which append raw pip output for Python; +// Core never does). The npm, system package, file and Skill labels are unverified. +// A script step without a reported exit status keeps the generic reason. +func (f ProvisioningFailure) reason() string { + label := map[string]string{ + ProvisioningPythonPackages: "Python package installation", + ProvisioningNPMPackages: "npm package installation", + ProvisioningSystemPackages: "System package installation", + }[f.Step] + if f.Step == ProvisioningSetupCommand && f.Index >= 0 { + label = fmt.Sprintf("setup_commands[%d]", f.Index) + } + switch { + case label != "" && f.ExitCode > 0 && f.ExitCode < 256: + return fmt.Sprintf("Failed to provision environment: script %q failed with exit code %d", label, f.ExitCode) + case f.Step == ProvisioningInitialFile: + return "Failed to provision environment: initial file installation failed" + case f.Step == ProvisioningSkill: + return "Failed to provision environment: Skill installation failed" + } + return provisioningFailureReason +} + +// EnvironmentFailure is a hosted Environment's recorded provisioning failure. It +// makes the Session failed with this reason and last activity time. +type EnvironmentFailure struct { + Reason string `json:"reason"` + FailedAt time.Time `json:"failed_at"` +} + +func environmentFailure(row sqlc.Environment) *EnvironmentFailure { + if row.Status != "failed" || !row.FailureReason.Valid || !row.FailedAt.Valid { + return nil + } + return &EnvironmentFailure{Reason: row.FailureReason.String, FailedAt: row.FailedAt.Time} +} + // terminateRuntimeEnvironment participates in the allocation's Session transaction. // Public expiry does not assert compute removal or invent an expired SSE variant. -func terminateRuntimeEnvironment(ctx context.Context, q *sqlc.Queries, current sqlc.GetRuntimeAllocationRow) error { +// A first failure records the hosted provisioning failure; an already terminal +// Environment only settles remaining input, without repeating events. +func terminateRuntimeEnvironment(ctx context.Context, q *sqlc.Queries, current sqlc.GetRuntimeAllocationRow, reason string, cancel func() error) error { row, err := q.GetSessionEnvironment(ctx, sqlc.GetSessionEnvironmentParams{TenantID: current.TenantID, ID: current.SessionID}) if err != nil { return err } - if row.Environment.Status != "expired" && row.Environment.Status != "failed" { - if current.Expired { - err = q.SetEnvironmentConnectionStatus(ctx, sqlc.SetEnvironmentConnectionStatusParams{ID: row.Environment.ID, Status: "expired"}) - } else { - err = recordEnvironmentConnection(ctx, q, row, "failed") + terminal := row.Environment.Status == "expired" || row.Environment.Status == "failed" + if !terminal && !current.Expired { + return failHostedEnvironment(ctx, q, row, current.SessionID, reason, cancel) + } + return withEnvironmentInputActivity(ctx, q, current.SessionID, func() error { + if !terminal { + if err := q.SetEnvironmentConnectionStatus(ctx, sqlc.SetEnvironmentConnectionStatusParams{ID: row.Environment.ID, Status: "expired"}); err != nil { + return err + } } - if err != nil { + if err := q.FailSessionEnvironmentInput(ctx, current.SessionID); err != nil { return err } + return cancel() + }) +} + +// failHostedEnvironment records, in the caller's transaction and in the observed +// official order, agent.session.environment.failed, an error event carrying the +// safe reason, then one agent.session.failed snapshot. The snapshot captures the +// settled input activity, Usage and the failure, matching later Session reads. +// Pending input settles as failed exactly as before. +func failHostedEnvironment(ctx context.Context, q *sqlc.Queries, row sqlc.GetSessionEnvironmentRow, session pgtype.UUID, reason string, cancel func() error) error { + failedAt, err := q.RecordEnvironmentFailure(ctx, sqlc.RecordEnvironmentFailureParams{ID: row.Environment.ID, FailureReason: pgtype.Text{String: reason, Valid: true}}) + if err != nil { + return err + } + if err := recordEnvironmentState(ctx, q, row, "failed"); err != nil { + return err + } + if err := q.FailSessionEnvironmentInput(ctx, session); err != nil { + return err + } + if err := cancel(); err != nil { + return err + } + activity, pending, err := environmentInputState(ctx, q, session) + if err != nil { + return err + } + usage, err := q.SessionTokenUsage(ctx, session) + if err != nil { + return err + } + if err := recordSessionChange(ctx, q, session, SessionChange{Event: v1.SessionEvent{ + Type: "error", Error: &v1.StreamError{Type: "environment_error", Code: "sandbox_error", Message: reason}, + }}); err != nil { + return err } - return q.FailSessionEnvironmentInput(ctx, current.SessionID) + return recordSessionChange(ctx, q, session, SessionChange{ + Event: v1.SessionEvent{Type: "agent.session.failed"}, + EnvironmentInputActivity: activity, EnvironmentFailure: &EnvironmentFailure{Reason: reason, FailedAt: failedAt.Time}, + SessionUsage: usage, Settled: !pending, + }) } diff --git a/services/agents-api/internal/store/runtime_environment_terminal_test.go b/services/agents-api/internal/store/runtime_environment_terminal_test.go index dc15cb065..33ea92238 100644 --- a/services/agents-api/internal/store/runtime_environment_terminal_test.go +++ b/services/agents-api/internal/store/runtime_environment_terminal_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + v1 "github.com/MiniMax-AI-Dev/parsar/contracts/agents-api/v1" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/device" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -43,6 +44,10 @@ func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testin if err != nil || ended.Environment.Status != status || ended.LastTurn != nil || ended.EnvironmentInputActivity == nil || ended.EnvironmentInputActivity.Status != "failed" || ended.EnvironmentInputActivity.Failure != "environment_unavailable" { t.Fatal("terminal projection", ended, err) } + // Only a hosted failure records a provisioning failure, with the generic reason. + if failure := ended.EnvironmentFailure; expired != (failure == nil) || !expired && (failure.Reason != provisioningFailureReason || failure.FailedAt.IsZero()) { + t.Fatal("terminal failure projection", failure) + } if _, ok, err := s.GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { t.Fatal("terminal credential remained usable", err) } @@ -50,7 +55,7 @@ func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testin if err != nil || failed.State != EnvironmentInputFailed || len(failed.Receipts) != 0 || failed.SettledAt == nil || !failed.Deadline.Equal(reservation.Deadline) { t.Fatal("late preparation resurrected failed input", failed, err) } - if _, err := s.ReserveEnvironmentInput(t.Context(), tenant, session.ID, "new", []Input{messageInput("later")}); !errors.Is(err, ErrEnvironmentUnavailable) { + if _, err := s.ReserveEnvironmentInput(t.Context(), tenant, session.ID, "new", []Input{messageInput("later")}); !errors.Is(err, ErrEnvironmentUnavailable) || expired == errors.Is(err, ErrHostedEnvironmentFailed) { t.Fatal("terminal environment admitted new input", err) } if _, err := s.CreateSession(t.Context(), tenant, input); err != nil { @@ -60,15 +65,31 @@ func TestManagedEnvironmentTerminationSettlesInputAndPreservesIdentity(t *testin if err != nil { t.Fatal(err) } - expected := 2 + expected := 3 if expired { expected = 1 } if len(events) != expected || events[len(events)-1].Event.Type != "agent.session.failed" { t.Fatal("wrong terminal events", events) } - if !expired && (events[0].Event.Type != "agent.session.environment.failed" || events[0].Event.Environment.Error == nil) { - t.Fatal("missing safe environment failure", events) + last := events[len(events)-1] + if activity := last.EnvironmentInputActivity; activity == nil || activity.Status != "failed" || activity.Failure != "environment_unavailable" || !last.Settled { + t.Fatal("pending input settlement changed", last) + } + if !expired { + environment, failure := events[0].Event, events[1].Event.Error + if environment.Type != "agent.session.environment.failed" || environment.Environment.Error == nil || + *environment.Environment.Error != (v1.StreamError{Type: "environment_error", Code: "environment_connection_failed", Message: "The environment failed to connect."}) { + t.Fatal("missing safe environment failure", events) + } + if events[1].Event.Type != "error" || failure == nil || *failure != (v1.StreamError{Type: "environment_error", Code: "sandbox_error", Message: provisioningFailureReason}) { + t.Fatal("missing safe error event", events) + } + if snapshot := last.EnvironmentFailure; snapshot == nil || snapshot.Reason != ended.EnvironmentFailure.Reason || !snapshot.FailedAt.Equal(ended.EnvironmentFailure.FailedAt) { + t.Fatal("failed snapshot differs from Session reads", last.EnvironmentFailure, ended.EnvironmentFailure) + } + } else if last.EnvironmentFailure != nil { + t.Fatal("expiry recorded a provisioning failure", last) } cursor, _ := s.SessionEventCursor(t.Context(), tenant, session.ID) if _, err := writer.RequestRuntimeCleanup(t.Context(), owner); err != nil { @@ -127,3 +148,28 @@ func TestManagedEnvironmentFailureRollsBackWithSessionEvent(t *testing.T) { t.Fatal("partial credential revocation", err) } } + +// Reasons contain only a fixed label and an exit status. Setup and Python labels +// match official samples; npm, system, file and Skill labels are unverified. +func TestProvisioningFailureReasons(t *testing.T) { + for failure, want := range map[ProvisioningFailure]string{ + {Step: ProvisioningSetupCommand, Index: 0, ExitCode: 3}: `Failed to provision environment: script "setup_commands[0]" failed with exit code 3`, + {Step: ProvisioningSetupCommand, Index: 12, ExitCode: 1}: `Failed to provision environment: script "setup_commands[12]" failed with exit code 1`, + {Step: ProvisioningPythonPackages, ExitCode: 1}: `Failed to provision environment: script "Python package installation" failed with exit code 1`, + {Step: ProvisioningNPMPackages, ExitCode: 1}: `Failed to provision environment: script "npm package installation" failed with exit code 1`, + {Step: ProvisioningSystemPackages, ExitCode: 100}: `Failed to provision environment: script "System package installation" failed with exit code 100`, + {Step: ProvisioningInitialFile}: "Failed to provision environment: initial file installation failed", + {Step: ProvisioningSkill}: "Failed to provision environment: Skill installation failed", + // Missing or impossible statuses, unknown steps and old receipts stay generic. + {Step: ProvisioningSetupCommand, Index: 0}: provisioningFailureReason, + {Step: ProvisioningSetupCommand, Index: -1, ExitCode: 3}: provisioningFailureReason, + {Step: ProvisioningPythonPackages, ExitCode: 256}: provisioningFailureReason, + {Step: ProvisioningNPMPackages, ExitCode: -9}: provisioningFailureReason, + {Step: "configure", ExitCode: 1}: provisioningFailureReason, + {}: provisioningFailureReason, + } { + if got := failure.reason(); got != want || len(got) > 256 { + t.Errorf("%+v: %q", failure, got) + } + } +} diff --git a/services/agents-api/internal/store/scheduling.go b/services/agents-api/internal/store/scheduling.go index c6fc8442c..8636fbc7b 100644 --- a/services/agents-api/internal/store/scheduling.go +++ b/services/agents-api/internal/store/scheduling.go @@ -142,6 +142,7 @@ func readSessionActivity(ctx context.Context, q *sqlc.Queries, session Session) return session, err } session.Environment = &value + session.EnvironmentFailure = environmentFailure(environment.Environment) session.EnvironmentInputActivity, session.PendingInput, err = environmentInputState(ctx, q, id) if err != nil { return session, err diff --git a/services/agents-api/internal/store/session_events.go b/services/agents-api/internal/store/session_events.go index e2446e78c..e657d2f21 100644 --- a/services/agents-api/internal/store/session_events.go +++ b/services/agents-api/internal/store/session_events.go @@ -24,6 +24,9 @@ type SessionChange struct { SessionUsage json.RawMessage `json:"session_usage,omitempty"` RequiredActions []v1.FunctionCallAction `json:"required_actions,omitempty"` EnvironmentInputActivity *EnvironmentInputActivity `json:"environment_input_activity,omitempty"` + // EnvironmentFailure is set on the agent.session.failed snapshot of a hosted + // provisioning failure, which also ends live event streams. + EnvironmentFailure *EnvironmentFailure `json:"environment_failure,omitempty"` // Settled marks an idle or failed snapshot recorded when a Turn ends, or when // the latest input reservation stops being pending (expired, cancelled or // failed). A reservation made while the ending Turn captured Artifacts can diff --git a/services/agents-api/internal/store/sessions.go b/services/agents-api/internal/store/sessions.go index 064990247..d627401d3 100644 --- a/services/agents-api/internal/store/sessions.go +++ b/services/agents-api/internal/store/sessions.go @@ -47,6 +47,9 @@ type Session struct { RequiredActions []v1.FunctionCallAction Environment *Environment EnvironmentInputActivity *EnvironmentInputActivity + // EnvironmentFailure is the recorded provisioning failure of a failed hosted + // Environment. It makes the Session failed and is terminal. + EnvironmentFailure *EnvironmentFailure // PendingInput reports that the latest input reservation, read once no Turn // is active or newer, can still start a Turn. It only supports settlement // checks and is never rendered. diff --git a/services/agents-api/migrations/000061_environment_failure.sql b/services/agents-api/migrations/000061_environment_failure.sql new file mode 100644 index 000000000..cda1a0d80 --- /dev/null +++ b/services/agents-api/migrations/000061_environment_failure.sql @@ -0,0 +1,18 @@ +-- +goose Up +-- A failed hosted Environment keeps the public reason and time of its +-- provisioning failure. Core composes the reason from a fixed step label and an +-- exit status; Runtime output is never stored. Earlier failures keep NULL. +ALTER TABLE environments + ADD COLUMN failure_reason text, + ADD COLUMN failed_at timestamptz, + ADD CONSTRAINT environment_failure_recorded CHECK ( + (failure_reason IS NULL) = (failed_at IS NULL) + AND (failed_at IS NULL OR status = 'failed') + AND char_length(failure_reason) <= 256 + ); + +-- +goose Down +ALTER TABLE environments + DROP CONSTRAINT environment_failure_recorded, + DROP COLUMN failed_at, + DROP COLUMN failure_reason; From 3448883ca236fb92d5e04951a2d73233e3d18f23 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 20:47:24 +0000 Subject: [PATCH 03/11] Deliver Session error events in the TypeScript client Error events other than Core's stream_interrupted now reach onEvent as AgentSessionErrorEvent before the failed snapshot instead of raising, and stream errors accept the pinned nullable param. --- packages/agents-client/src/client.test.ts | 44 +++++++++++++++++++++++ packages/agents-client/src/client.ts | 40 +++++++++++++++------ packages/agents-client/src/types.ts | 16 ++++++++- 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/packages/agents-client/src/client.test.ts b/packages/agents-client/src/client.test.ts index e3a96fe3a..d879ecce6 100644 --- a/packages/agents-client/src/client.test.ts +++ b/packages/agents-client/src/client.test.ts @@ -316,6 +316,50 @@ describe("OpenAIAgentsClient", () => { expect(cancelled).toBe(true); }); + it("delivers a hosted provisioning failure's error event before the failed snapshot", async () => { + const reason = 'Failed to provision environment: script "setup_commands[0]" failed with exit code 3'; + const environment = { ...hostedDadf64.session_environment, id: "environment" }; + const failed = { ...sessionResource(), environment, status: "failed", error: reason, last_active_at: 25 }; + const frames = [ + { + type: "agent.session.environment.failed", event_id: "evt_environment", session_id: "session", + environment: { + id: "environment", type: "openai_hosted", status: "failed", + error: { type: "environment_error", code: "environment_connection_failed", message: "The environment failed to connect." }, + }, + }, + { + type: "error", event_id: "evt_error", session_id: "session", + error: { type: "environment_error", code: "sandbox_error", message: reason, param: null }, + }, + { type: "agent.session.failed", event_id: "evt_failed", session: failed }, + ]; + const onEvent = vi.fn(); + const client = new OpenAIAgentsClient({ + fetch: recordingFetch(streamResponse(frames.map((frame) => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`)), []), + }); + + await expect(client.streamEvents("session", { onEvent })).resolves.toBeUndefined(); + expect(onEvent.mock.calls.map(([event]) => event.type)).toEqual(["agent.session.environment.failed", "error", "agent.session.failed"]); + expect(onEvent.mock.calls[1]?.[0]).toEqual(frames[1]); + expect(onEvent.mock.calls[2]?.[0]).toMatchObject({ session: { status: "failed", error: reason, last_active_at: 25 } }); + }); + + it.each([ + ["null param", { code: "stream_interrupted", type: "server_error", message: "safe", param: null }, 503], + ["invalid param", { code: "sandbox_error", type: "environment_error", message: "safe", param: 1 }, 502], + ["extra error field", { code: "sandbox_error", type: "environment_error", message: "safe", param: null, output: "private" }, 502], + ])("keeps in-band error validation with %s", async (_label, error, status) => { + const onEvent = vi.fn(); + const event = { type: "error", event_id: "evt_error", session_id: "session", error }; + const client = new OpenAIAgentsClient({ + fetch: recordingFetch(streamResponse([`event: error\ndata: ${JSON.stringify(event)}\n\n`]), []), + }); + + await expect(client.streamEvents("session", { onEvent })).rejects.toMatchObject({ status }); + expect(onEvent).not.toHaveBeenCalled(); + }); + it("creates a Session through chunked SSE and publishes its validated leading snapshot exactly once", async () => { const calls: FetchCall[] = []; const controller = new AbortController(); diff --git a/packages/agents-client/src/client.ts b/packages/agents-client/src/client.ts index 841acb436..6b4f2d47a 100644 --- a/packages/agents-client/src/client.ts +++ b/packages/agents-client/src/client.ts @@ -316,6 +316,8 @@ const maxSessionInputEvents = 64; const maxSessionInputRequestBytes = 1024 * 1024; const goWhitespaceOnlyPattern = /^[\u0009-\u000d\u0020\u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000]*$/u; const streamErrorFields = new Set(["code", "type", "message"]); +// Error events carry the pinned SessionError, whose param is null when unset. +const sessionErrorFields = new Set([...streamErrorFields, "param"]); const environmentStateFields = new Set(["id", "type", "status", "error"]); const snapshotEventFields = new Set(["type", "event_id", "session_id", "session"]); const turnEventFields = new Set(["type", "event_id", "session_id", "turn_id", "turn"]); @@ -1221,12 +1223,18 @@ function projectRuntimeObservationList(value: unknown, options?: PageOptions): R function projectStreamError(value: unknown): StreamError { if ( - !isRecord(value) || !exactFields(value, streamErrorFields) || + !isRecord(value) || !onlyFields(value, sessionErrorFields) || typeof value.code !== "string" || value.code === "" || typeof value.type !== "string" || value.type === "" || - typeof value.message !== "string" + typeof value.message !== "string" || + !(value.param === undefined || value.param === null || typeof value.param === "string") ) return invalidStreamEvent(); - return { code: value.code, type: value.type, message: value.message }; + return { + code: value.code, + type: value.type, + message: value.message, + ...(value.param === undefined ? {} : { param: value.param as string | null }), + }; } function requiredEventString(event: Record, field: string, allowEmpty = false): string { @@ -1431,6 +1439,14 @@ function projectStreamEventSession( } as SessionEvent; } + // A Session failure, such as a hosted Environment that failed to provision. + // Core's stream_interrupted error never reaches this projection. + if (event.type === "error") { + if (!exactFields(value, errorEventFields)) return invalidStreamEvent(); + const sessionId = eventSessionId(value, expectedSessionId, true)!; + return { ...base, session_id: sessionId, error: projectStreamError(value.error) } as SessionEvent; + } + if (event.type.startsWith("agent.session.environment.")) { const status = event.type.slice("agent.session.environment.".length); if (!new Set(["pending", "ready", "connected", "disconnected", "failed"]).has(status)) { @@ -1505,13 +1521,17 @@ async function consumeEventStream( return invalidStreamEvent("Agent Core returned an event for a different Session."); } const streamError = projectStreamError(event.error); - throw new AgentCoreError( - "Agent Core interrupted the live event stream. Reconnect and retrieve durable state.", - 503, - streamError.code, - null, - streamError.type, - ); + // Only Core's own interruption ends delivery. Other error events report a + // Session failure, delivered in order before agent.session.failed. + if (streamError.code === "stream_interrupted") { + throw new AgentCoreError( + "Agent Core interrupted the live event stream. Reconnect and retrieve durable state.", + 503, + streamError.code, + null, + streamError.type, + ); + } } options.onParsedEvent(event); }); diff --git a/packages/agents-client/src/types.ts b/packages/agents-client/src/types.ts index d0e47acdf..3083d1891 100644 --- a/packages/agents-client/src/types.ts +++ b/packages/agents-client/src/types.ts @@ -629,6 +629,8 @@ export interface StreamError { code: string; type: string; message: string; + /** Present on error events (null when unset); Environment state errors omit it. */ + param?: string | null; } export type SessionEnvironmentStatus = "pending" | "ready" | "connected" | "disconnected" | "failed"; @@ -700,7 +702,19 @@ export interface UnknownSessionEvent extends SessionEventBase { [key: string]: unknown; } -export type SessionEvent = AgentSessionEnvironmentEvent | KnownSessionEvent | UnknownSessionEvent; +/** + * A Session failure reported in the event stream, such as a hosted Environment + * that failed to provision (type environment_error, code sandbox_error). The + * agent.session.failed snapshot follows it. Core's own stream interruption is + * raised as an AgentCoreError instead. + */ +export interface AgentSessionErrorEvent extends SessionEventBase { + type: "error"; + session_id: string; + error: StreamError; +} + +export type SessionEvent = AgentSessionEnvironmentEvent | AgentSessionErrorEvent | KnownSessionEvent | UnknownSessionEvent; export interface AgentDeleted { id: string; From 1fa024e229c81d1352662115820c06398e6043e2 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 20:48:02 +0000 Subject: [PATCH 04/11] Document hosted initialization failure alignment Record rows H1-H8 with the HI-01..04 evidence, the initialization failure lifecycle and reasons, the failure event sequence and stream end, the HF evidence register code and the updated Session operation rows. --- CONTRIBUTING.md | 14 +++- contracts/agents-api/environment-templates.md | 37 ++++++++++- contracts/agents-api/history-events-usage.md | 34 ++++++++++ .../official-semantics-alignment.md | 64 +++++++++++++++++++ contracts/agents-api/operation-evidence.md | 13 ++-- services/agents-api/README.md | 4 +- 6 files changed, 156 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c723330b8..14f04ff33 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -440,6 +440,15 @@ This bounds extra maintenance work but does not bypass capacity, a busy lifecycl gate or multi-page scheduling, and does not guarantee a resume deadline. A recovered or uncertain running installation fails and uses existing cleanup, without replaying writes. +A hosted provisioning failure is terminal for its Session. The allocation's cleanup +transaction stores a safe reason with the failed Environment and records +`environment.failed`, `error` and one `agent.session.failed`; Session reads derive +`failed`, that reason and the failure time from the same record, and live streams end +after the failed event. The initializer reports only the integer exit status of a +failed sandboxed step. Core composes the reason from a fixed step label and that status, +never from command, package-manager or file output; unknown effects, timeouts and +receipts without a status keep a generic reason. New input then gets the observed 409 +`conflict_error`; expiry and pending-input settlement keep their behavior. Completed environments never reinstall initial files on reconnect or native recovery. Provider RunCommand carries bounded stdin, not confidential argv. Only fixed trusted initializers may run with Runtime authority. User setup and package install hooks @@ -1266,8 +1275,9 @@ admission deadline plus response grace. Request/observer disconnect stops waitin not the durable reservation or execution; retries keep the original identity and deadline. The Worker remains the readiness, promotion and Start owner. Local failure mapping uses 409 `environment_input_expired` / `environment_input_cancelled`, 503 -`execution_unavailable` for ownership loss, and existing 404 for deletion. Exact -hosted failure status/body and pending-input crash recovery remain unverified. +`execution_unavailable` for ownership loss, and existing 404 for deletion. New input +after a hosted provisioning failure returns the observed 409 `conflict_error`; other +exact hosted failure statuses/bodies and pending-input crash recovery remain unverified. Principal acceptance must use public Session creation and input against the built standalone service, including a wait exceeding its ordinary 30-second write timeout, real remote commands/files and a second native-history Turn. Private provisioning diff --git a/contracts/agents-api/environment-templates.md b/contracts/agents-api/environment-templates.md index 251790f6c..3b5bdf402 100644 --- a/contracts/agents-api/environment-templates.md +++ b/contracts/agents-api/environment-templates.md @@ -415,7 +415,10 @@ version-1 JSON operation on stdin. It configures read-only tool env under and runs ordered commands through distro bubblewrap. The fixed mount/process map excludes daemon credentials, native history and staging. User values are applied inside isolation, never to the launcher. Receipt and process exit must both confirm -completion; child output is discarded because it can contain secrets. +completion; child output is discarded because it can contain secrets. A failed step +that ran inside that isolation (a setup command or a package manager) adds only its +integer `exit_code` to the failed receipt; see +[Initialization failure](#initialization-failure--september-23). Runtime receives a `tool_environment` execution flag, without template identity or provider information. Adapters validate the common files and apply them in their @@ -430,6 +433,38 @@ The adapter verifies the required trusted managed hook before preparation and stops the Turn on an observed failed hook. Earlier command effects may already exist; this is not an atomic hook-failure prevention guarantee. +## Initialization failure — September 23 + +When a hosted Environment fails to provision, Core now reports it the way the +official service does (evidence and rows H1–H8 in +[official semantics](official-semantics-alignment.md#hosted-initialization-failure--september-23)). +In the allocation's cleanup transaction, Core marks the Environment failed and +records, in order, `agent.session.environment.failed`, an `error` event and one +`agent.session.failed`. The Session reads `failed` with the safe reason as `error` +and the failure time as `last_active_at`; live streams end after the failed event. +New input on that Session returns 409 `conflict_error` "the hosted environment +failed to provision". Pending input settles as failed exactly as before. + +The reason names only the failed step and its exit status: + +| Step | Reason | +| --- | --- | +| Setup command `i` | `Failed to provision environment: script "setup_commands[i]" failed with exit code N` (observed) | +| Python packages | `... script "Python package installation" failed with exit code N` (observed label; the official reason appends raw pip output, Core never does) | +| npm or system packages | `... script "npm package installation"` / `"System package installation"` `failed with exit code N` (unverified) | +| Initial file or Skill with a confirmed failed write | `Failed to provision environment: initial file installation failed` / `Skill installation failed` (unverified) | +| Anything else | `Failed to provision environment: initialization did not complete` | + +"Anything else" covers timeouts, the thirty-minute budget, unknown effects, +missing or malformed receipts, receipts without `exit_code` from Runtime images +built before this change, Plugin and capability installation, bootstrap +rejection and Core restart during initialization. Core accepts an exit status +only from a strict version-1 failed receipt with process status 1 and no stderr, +as an integer from 1 to 255, and the Store composes the reason from a fixed label +and integers. Commands, env values, package names, paths and any process output +therefore never reach the reason, events, logs or responses. The failed step is +not retried and later steps do not run. + ## System packages `packages.system` accepts package names for the Runtime's Debian apt repositories, diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index 53cf9a24f..6bb2a2b66 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -338,3 +338,37 @@ the TypeScript client and Web unit tests. `make openapi` adds only `x-nullable` Item `phase` and event `output_index`; `make sqlc-generate` changes `SessionTokenUsage` and adds the internal `SessionMeasuredTokenUsage`. The native pinned-SDK scripts updated for these shapes run only with a native daemon, and live model acceptance is recorded separately. + +## Hosted initialization failure events, 2026-09-23 + +Evidence: campaign scan 6 HI-01..04 (private +`~/.parsar/remediation/20260923/campaign-scan-6/hosted-init/`, raw frames +`official/007-S2-events.json` and `009-S3-events.json`); rows H1–H8 are in +[official semantics](official-semantics-alignment.md#hosted-initialization-failure--september-23). + +- **Order.** A hosted Environment that fails to provision records + `agent.session.environment.failed`, `error` and `agent.session.failed` in one + transaction, as officially observed. The official streams showed no + `environment.pending` event; Core records none either. +- **Payloads.** `environment.error` is `{type: environment_error, code: + environment_connection_failed, message: "The environment failed to connect."}`. + The `error` event carries the pinned `SessionError`: `{type: environment_error, + code: sandbox_error, message: , param: null}`. Every `error` event + now includes `param` (null when unset), including Core's own + `stream_interrupted`. The `agent.session.failed` snapshot has `status: failed`, + the reason as `error`, `required_actions: []` and the failure time as + `last_active_at`, identical to later retrieve and list reads. Pending input + settled by the failure is captured in the same snapshot. +- **Stream lifetime.** GET and creation streams end right after that + `agent.session.failed`, as the official GET stream did. This changes the + earlier rule that GET streams never end on their own, for this terminal case + only: a Turn failure leaves GET streams open because the Session can continue, + and a GET stream opened after the failure stays open (not observed officially). +- **Client.** The TypeScript client still raises `stream_interrupted` as an + `AgentCoreError`, and now delivers other `error` events to `onEvent` as + `AgentSessionErrorEvent`, before the failed snapshot. It accepts an optional + nullable `param` on stream errors. Core Web renders the failed Session and its + error from the snapshot and ignores the error event. + +Environments that failed before migration `000061` have no recorded reason; they +keep their earlier projection and events. diff --git a/contracts/agents-api/official-semantics-alignment.md b/contracts/agents-api/official-semantics-alignment.md index 1bb4eddcb..1e98d8387 100644 --- a/contracts/agents-api/official-semantics-alignment.md +++ b/contracts/agents-api/official-semantics-alignment.md @@ -725,3 +725,67 @@ creation-stream and live events, deletion and retries. The pinned-SDK scripts `official_mcp.py` and `official_mcp_credentials.py` assert the omitted origin, the projection and the error fields. Real Core acceptance is recorded separately by the coordinator. + +## Hosted initialization failure — September 23 + +Hosted Environments that fail to provision now surface the failure as the +official service does, from Core main `e1970fd7`. Evidence is HI-01..04 in the +campaign scan recorded privately in +`~/.parsar/remediation/20260923/campaign-scan-6/hosted-init/findings.json`, with +raw official records under `official/`: `006-S2-create-setup-exit3`, +`007-S2-events`, `009-S3-events`, `021-S2-env-after-failed`, +`024-S2-session-after-failed`, `025-S3-session-after-failed`, +`026-S2-input-after-failure`, `027-S3-input-after-failure`, `038-S2-delete` and +`041-S3-delete`. Two owned `openai_hosted` Sessions without a Turn failed, one on a +setup command that echoed a value and exited 3, one on a nonexistent Python +package; both were deleted. The batch plan is +`~/.parsar/remediation/20260924/hosted-init-failure/PLAN.md`. + +| Row | Case | Core behavior | +| --- | --- | --- | +| H1 | Hosted initialization fails in any step | One transaction records the Environment failure, `agent.session.environment.failed`, `error` and one `agent.session.failed`. The Session reads `status: failed`, the stored safe reason as `error`, `required_actions: []` and the failure time as `last_active_at`; retrieve, list and the event snapshot agree. Pending input reserved for the Environment settles as failed exactly as before, captured in the same snapshot. | +| H2 | `environment.failed` payload | `error` is `{type: environment_error, code: environment_connection_failed, message: "The environment failed to connect."}`. | +| H3 | `error` event | `{type: environment_error, code: sandbox_error, message: , param: null}`. | +| H4 | Reason | `Failed to provision environment: script "setup_commands[i]" failed with exit code N`, and `script "Python package installation"` for Python packages; the official Python reason also appends raw pip output, which Core never copies. npm, system package, initial file and Skill labels are unverified. Other failures use `Failed to provision environment: initialization did not complete`; see the [initialization lifecycle](environment-templates.md#initialization-failure--september-23). | +| H5 | Live SSE | GET and creation streams end right after that `agent.session.failed`. | +| H6 | Later `events.create` | 409 `conflict_error`/`conflict_error` "the hosted environment failed to provision", param null. Expired Environments, and input already waiting when the Environment failed, keep 409 `environment_unavailable`. | +| H7 | Delete | 200 `agent.session.deleted`, as officially, then 404. Deletion while provisioning is unchanged (HI-05 awaits a decision). | +| H8 | Unchanged | `self_hosted` and `none` Environments, successful initialization and its timing, the two-minute step limit (HI-06), expiry and tenant isolation. | + +Decisions: + +- **No output.** The shared Runtime initializer adds only an integer `exit_code` + to its failed receipt, and only for a step run inside its bwrap isolation; + Runtime helpers, signals and other errors keep the generic receipt, and the + exception is never serialized. Core accepts the status only from a strict + version-1 failed receipt with process status 1 and no stderr, as an integer + from 1 to 255. The Store composes the reason from a fixed step label and + integers, so commands, env values, package names, paths and process output + cannot reach the reason, events, logs or responses. +- **Storage.** The additive migration `000061_environment_failure.sql` adds the + nullable `environments.failure_reason` and `failed_at`; a check ties them to + `status = failed` and bounds the reason to 256 characters. Environments that + failed earlier keep NULL and their previous projection and events; new input + on them gets the H6 409. +- **Runtime images.** A Runtime image built before this change reports no + `exit_code`; its failures use the generic reason and otherwise follow H1–H7. + The Codex, Claude and MiniMax Code images must be rebuilt for exit statuses. +- **Scope of the terminal state.** Only a recorded hosted provisioning failure + makes the Session terminal. A Turn failure still leaves GET streams open, and + a GET stream opened after the failure stays open; that case was not observed. +- **Clients.** Every `error` event now carries `param`. The TypeScript client + delivers error events other than Core's `stream_interrupted` to `onEvent` + before the failed snapshot, instead of raising them. Core Web already renders + the failed Session, its error and the blocked input. + +Go store tests on a dedicated PostgreSQL database drive the managed Worker with a +controlled Provider through setup exit statuses (first and later command), +Python packages, a receipt without `exit_code`, an unknown effect, raw output +instead of a receipt and a failed initial file write. They check the Session +read, list, the exact three events and snapshot, the H6 rejection, pending-input +settlement, tenant B and the absence of a canary. A real-PostgreSQL HTTP test +checks retrieve, list, the live GET stream and its end, the exact 409, tenant B +404s, delete and the canary in every body. Go API and contract tests pin the +projection, stream lifetime, wire shapes and error mapping; Python tests pin the +initializer receipt, and the TypeScript client and Web unit tests pass. Live +Docker acceptance is recorded separately by the coordinator. diff --git a/contracts/agents-api/operation-evidence.md b/contracts/agents-api/operation-evidence.md index 395ba2494..82de16bf8 100644 --- a/contracts/agents-api/operation-evidence.md +++ b/contracts/agents-api/operation-evidence.md @@ -1,6 +1,6 @@ # Pinned operation evidence inventory — 2026-09-23 -Baseline inventory of main `b5715912f09333e2b4449ec6f0eecaabce44c9b7`. The Session admission batch below updates creation and metadata validation, the list query tolerance batch (L) updates list and resource query handling, the validation error batch (X) updates field error codes/params, malformed path IDs and U+0000 handling, the Environment Files wire batch (G) updates Files.create/list status, envelope, query, path and empty-page behavior, and the creation stream settlement batch (J) updates creation SSE lifetime/snapshot, terminal Turn usage and Turn start order, and the Session deletion batch (Z) updates the deletion lifecycle, and the Agent configuration validation batch (M) updates saved and inline Agent configuration errors, the whitespace input batch (P) admits whitespace-only message text, the list cursor error batch (CE) updates unresolved `after` cursor errors on every list, the input conflict batch (CF) gives every 409 type `conflict_error` and aligns Session input conflicts and tool result target errors, and the saved web_search batch (SW) saves every pinned `web_search` mode while Session admission keeps rejecting enabled search, and the workspace file write batch (FW) aligns Files.create parent creation, no-replacement and the inline size bound, and the item serialization batch (SR) aligns Item/event null fields, assistant message event framing, reasoning keys and the Session usage rule, and the MCP credential selection batch (MV) saves an omitted HTTP MCP origin as `service`, projects the implicitly selected Session credential and aligns selection errors; historical evidence retains its original revision and scope. This inventory guides repeated qualification and does not assert complete compatibility. +Baseline inventory of main `b5715912f09333e2b4449ec6f0eecaabce44c9b7`. The Session admission batch below updates creation and metadata validation, the list query tolerance batch (L) updates list and resource query handling, the validation error batch (X) updates field error codes/params, malformed path IDs and U+0000 handling, the Environment Files wire batch (G) updates Files.create/list status, envelope, query, path and empty-page behavior, and the creation stream settlement batch (J) updates creation SSE lifetime/snapshot, terminal Turn usage and Turn start order, and the Session deletion batch (Z) updates the deletion lifecycle, and the Agent configuration validation batch (M) updates saved and inline Agent configuration errors, the whitespace input batch (P) admits whitespace-only message text, the list cursor error batch (CE) updates unresolved `after` cursor errors on every list, the input conflict batch (CF) gives every 409 type `conflict_error` and aligns Session input conflicts and tool result target errors, and the saved web_search batch (SW) saves every pinned `web_search` mode while Session admission keeps rejecting enabled search, and the workspace file write batch (FW) aligns Files.create parent creation, no-replacement and the inline size bound, and the item serialization batch (SR) aligns Item/event null fields, assistant message event framing, reasoning keys and the Session usage rule, and the MCP credential selection batch (MV) saves an omitted HTTP MCP origin as `service`, projects the implicitly selected Session credential and aligns selection errors, and the hosted initialization failure batch (HF) aligns failed hosted provisioning: Session status/error, failure events, stream end and later-input 409; historical evidence retains its original revision and scope. This inventory guides repeated qualification and does not assert complete compatibility. Baseline: `contracts/agents-api/upstream.json`, SDK **3.13.0**, upstream commit **d7c41efee1b0802b79f3f88a678ef2052b06e9ce**, `OpenAI-Beta: agents=v1`. AGENTS.md and relevant CONTRIBUTING.md compatibility, ownership and evidence rules govern this inventory. @@ -51,6 +51,7 @@ Repository paths below are relative to the inspected worktree; private evidence | SW | [Saved web_search modes](official-semantics-alignment.md#saved-web_search-modes--september-23); private `~/.parsar/remediation/20260923/campaign-scan-3/subagents-tools/findings.json` TV-05 (W01/W02) and `~/.parsar/remediation/20260923/saved-web-search/official/{results.json,ledger.jsonl}`: four owned Agents (all deleted) with create records `type-only`, `mode-null`, `mode-cached`, `mode-cached-full`, update records `update-disabled`, `update-omitted-low`, `update-live-domains-empty` and `retrieve`, plus a second probe of two Agents (deleted, 404 confirmed) with `location-partial-omitted` and `location-empty`, without a Session or model. Rows W1–W8 of that section. Go handler, real-PostgreSQL HTTP exact-bytes/no-write/tenant B, pinned-SDK, TypeScript client and Web tests; independent acceptance is recorded with the batch. | | SR | [Item serialization](history-events-usage.md#item-serialization-2026-09-23); private `~/.parsar/remediation/20260923/campaign-scan-2/events-tools/findings.json` EVT-09, EVT-10, EVT-13 with raw frames `official/streams-s1..s4.json` and Items pages `official/calls-s2.json` `s2-items-after-t1`, `calls-s4.json` `s4-items`; `campaign-scan-1/sessions/findings.json` SES-23/25 and `campaign-scan-1/vaults-agents/findings.json` VA-11; `campaign-scan-5/sessions-turns/findings.json` ST-03 with raw `official/calls.json`; live-kit EVT-24 `creation-stream-settlement/acceptance/candidate-evidence/codex-kimi/attempt-1/r1-*.json`. Rows S1–S8 of that section. Go contract, API and real-PostgreSQL store tests, Codex adapter usage tests, the pinned-SDK official client suite and TypeScript client/Web tests; live acceptance is recorded with the batch. | | MV | [MCP origin and credential selection](official-semantics-alignment.md#mcp-origin-and-credential-selection--september-23); private `~/.parsar/remediation/20260923/campaign-scan-6/mcp-vaults/findings.json` MV-01..03 with raw records in `official-ledger.jsonl` (`AG1-minimal-and-nulls`, `S1-T1-create-stream`, `S1-after-c1-delete-session-get`, `S1-final-session-get`, `ERR-UNATTACHED`, `ERR-URL-MISMATCH`, `ERR-AMBIGUOUS`, `ERR-CREDENTIAL-BOGUS`, `ERR-VAULT-BOGUS`): owned Agents, three owned Sessions, two Vaults and four Credentials, all deleted. Rows M1–M9 of that section. Go API/store and real-PostgreSQL tenant A/B HTTP tests with a no-write digest, pinned-SDK official client scripts; live acceptance is recorded with the batch. | +| HF | [Hosted initialization failure](official-semantics-alignment.md#hosted-initialization-failure--september-23); private `~/.parsar/remediation/20260923/campaign-scan-6/hosted-init/findings.json` HI-01..04 with raw records under `official/` (`006-S2-create-setup-exit3`, `007-S2-events`, `009-S3-events`, `021-S2-env-after-failed`, `024-S2-session-after-failed`, `025-S3-session-after-failed`, `026-S2-input-after-failure`, `027-S3-input-after-failure`, `038-S2-delete`, `041-S3-delete`): two owned `openai_hosted` Sessions without a Turn (setup exit 3, nonexistent Python package), both deleted. Rows H1–H8 of that section; HI-05/06 stay deferred. Python initializer receipt tests, Go contract/API tests, real-PostgreSQL managed-Worker store tests with a controlled Provider and an HTTP test with tenant B and a canary, TypeScript client and Web tests; live Docker acceptance is recorded with the batch. | ## Per-operation evidence matrix @@ -64,12 +65,12 @@ Paths in the appendix include `/v1`. SDK names here omit `client.`. `P` means pa | 4 | beta.agents.list | P: scoped cursor list; unknown keys ignored, limit 0/above 100 clamp; any unresolved cursor, malformed included, is the missing 404 | R `agent-list-empty-scoped`, `agent-list-limit101`; L VA-01/02/03/04/18; CE ERR-13 `cur-agents-random`, `cur-agents-othertype-session` | L DB `official_list_query.py` tenant A/B; CE DB cursor matrix tenant A/B | Core page capacity 100; no inferred official cap. Overflowing limits unsampled | | 5 | beta.agents.delete | P: resource deletion | R `cleanup-agent`, subsequent 404 | C DB delete/post-delete | Referenced/in-flight/repeated-delete exact parity | | 6 | beta.agents.sessions.create | P: JSON/live SSE 201, saved/inline frozen config, initial messages, native profiles; inline agent configuration errors use official fields with `agent.` params before the input requirement, and saved records with repeated tools or non-object schema roots reject admission; fresh creation SSE sends the JSON 201 projection, then ends right after the first idle recorded when a Turn ends or an input reservation stops being pending, or any failed, never sending later events; nothing admitted ends after `created`; a silent settlement ends after events up to the cursor read with a settled projection in one snapshot. A same-key stream retry returns 201, sends no events and ends at once; whitespace-only text is admitted and stored verbatim, while empty text/content/input keep the local 400; omitted/null HTTP MCP origin is `service`; MCP credential selection errors use the official status, code, null param and message after the input requirement, with one message for missing, foreign and unattached references, and write nothing | S `create-1/2.json`, `omitted-input.json`, `null-input.json`, `empty-array-input.json`, `retry-status-original/repeat.json`; H stream; J creation streams closed after idle, open through requires_action; M TV-01..04 `SC01`–`SC11`; P SES-01..03 whitespace-only string and message input 201, verbatim Item; MV MV-01 `S1-T1-create-stream`, MV-03 `ERR-UNATTACHED`, `ERR-URL-MISMATCH`, `ERR-AMBIGUOUS`, `ERR-CREDENTIAL-BOGUS`, `ERR-VAULT-BOGUS` | N Live none admission and retry; C Live three hosted profiles; D/T/K/I recorded additional workflows; J DB creation-stream lifetime/snapshot/retry; M DB inline and saved-override configuration errors without writes; P DB verbatim whitespace Items and unchanged empty-input 400 without writes; MV DB M1–M8 tenant A/B, byte-identical not-found bodies and no-write digest | Session admission batch removes idle `none` creation; local idempotent create still differs from two official IDs. Stream retry, self-hosted, hosted and no-input stream lifetimes are local; work drained before a silent-settlement read can still be sent. Many input/tool/environment combinations restricted; harness admission limits keep the local code (TV-06). Empty-input 400s keep the local code and message; `["", text]` parts are accepted but unobserved officially (SES-08); Claude SDK and MiniMax Code reject whitespace-only messages at admission as a declared native limitation (W6); the unknown-Vault message stays `Resource not found.`; a deleted selected credential still fails at dispatch rather than input (MV-04) | -| 7 | beta.agents.sessions.retrieve | P: persisted state, required actions, usage; malformed ID equals missing; `agent.reasoning` carries both keys, null when unset; usage is the recorded root Turn sum only when every root Turn has ended with known usage, otherwise null; an MCP tool without explicit `credential_id` shows the implicitly selected credential ID, also after deletion | S `retrieve-1.json`, `session-after-1.json`; H recovered state; X SES-28; SR SES-23, EVT-13, ST-03; MV MV-02 `S1-after-c1-delete-session-get`, `S1-final-session-get` | C Live history; T pending actions; H Core acceptance recorded; SR API/DB reasoning and usage rule; MV API/DB selected-credential projection before and after deletion | Complete statuses/actions/lifecycle timing; Claude/MiniMax public usage remains null; model-derived default effort not resolved (VA-11); queued-Turn usage unobserved (treated as not ended) | +| 7 | beta.agents.sessions.retrieve | P: persisted state, required actions, usage; malformed ID equals missing; `agent.reasoning` carries both keys, null when unset; usage is the recorded root Turn sum only when every root Turn has ended with known usage, otherwise null; an MCP tool without explicit `credential_id` shows the implicitly selected credential ID, also after deletion; a hosted Environment that failed to provision reads `failed` with its safe step/exit-status reason and failure time | S `retrieve-1.json`, `session-after-1.json`; H recovered state; X SES-28; SR SES-23, EVT-13, ST-03; MV MV-02 `S1-after-c1-delete-session-get`, `S1-final-session-get`; HF HI-01/04 `024-S2-session-after-failed`, `025-S3-session-after-failed` | C Live history; T pending actions; H Core acceptance recorded; SR API/DB reasoning and usage rule; MV API/DB selected-credential projection before and after deletion; HF DB/HTTP failure projection, tenant B | Complete statuses/actions/lifecycle timing; Claude/MiniMax public usage remains null; model-derived default effort not resolved (VA-11); queued-Turn usage unobserved (treated as not ended) | | 8 | beta.agents.sessions.update | P: metadata-only replacement/clear; metadata errors use official code and `metadata`/`metadata.` param | S `metadata-replace/null/empty/omit/invalid-value.json`; `update-agent.json` uses newer unpinned field | N Live completed Session metadata rejection/clear/isolation; recorded active controlled metadata coverage | Session admission batch changes empty update to observed official 400; Session agent update belongs to baseline upgrade, not fixed-pin operation gap | -| 9 | beta.agents.sessions.list | P: Agent filter, full envelope, cursor paging; unknown keys ignored, limit 0/above 100 clamp; any unresolved cursor, malformed included, is the missing 404; Sessions project the selected MCP credential as retrieve does | S `list-filter.json`, `list-empty-after.json`, `list-owned-cross-filter-cursor.json`, limit/order/unknown-query samples; L SES-10/11/15/16/17; CE ERR-01 `cur-sessions-malformed`, ERR-13/14 | C Live order/cursors/empty/tenant checks; L DB tenant A/B; CE DB cursor matrix tenant A/B; MV DB selected-credential projection | Core page capacity 100; official cap unknown. Eventual visibility sample is not a required delay | -| 10 | beta.agents.sessions.delete | P: deletion only of a durably idle or failed Session without required actions or pending input; a busy root Turn or pending reservation gives 409 `conflict_error` with no change (subagent child Turns and pending Environment file writes are not checked); owner repeat returns the same 200; owned managed cleanup, user compute retained | S cleanup files 200/deleted; retry-session active cleanup initially 409; Z SES-29 `q5-delete-repeat` 200, SES-30 `q5b-delete-while-in-progress` 409, `q5-delete-never-existed` 404, `q5-delete-while-running` 200 right after an `events.create` 202 | D recorded real cleanup; C cleanup separately recorded; Z DB matrix D1–D4 with no-write digest, admission lock race and pinned-SDK script | Core returns 409 right after an `events.create` 202 because it admits Turns synchronously (official 200); input awaiting its Environment cannot be cancelled publicly and is unobserved officially; physical purge/retention may end repeat idempotency; caller compute ownership preserved | -| 11 | beta.agents.sessions.events.create | P: 202/empty body, empty-array authenticated no-op, text/cancel/function admission; whitespace-only text is admitted and stored verbatim, empty text/content/input keep the local 400; input the Session cannot accept (result after cancellation, batch while input is pending) and changed results give 409 `conflict_error`/`conflict_error`; in an owned Session an unknown call or a call of another Turn gives 400 `invalid_request_error` without writes; missing/foreign Sessions stay 404; key reuse keeps local 409 `idempotency_conflict` with type `conflict_error` | S `second-turn-create.json`, `events-empty/null.json`; H second-input; P SES-04 two whitespace-only messages 202, SES-06/07 empty text/content/input 400; CF EVT-11 `s2-result-unknown-call`/`s2-result-unknown-turn` 400, EVT-12 `s2-result-changed-after-terminal`/`s2-result-after-cancel` 409, EVT-14 duplicate 202, ERR-22 `sessB-message-while-running` 409 | C Live real continuation/no-op; T qualified message/result/cancel workflows; P DB verbatim whitespace Items and no-write empty-input rejection; CF DB rows CF2–CF4 and CF6–CF9 with tenant B, no-write digest and unchanged pending action, pinned-SDK result script | Mixed prepared-environment batches, native receipt vs durable acceptance, cancel-before-result-publication timing; unqualified content/tools; empty-input error code/message differ; `["", text]` unobserved officially (SES-08); Claude SDK and MiniMax Code reject whitespace-only messages at admission as a declared native limitation (W6); the official asynchronous pending-input window (ERR-22) is not emulated; Core messages omit call/executor IDs; official order between pending-input and target errors unobserved | -| 12 | beta.agents.sessions.events.stream | P: live-only SSE, typed persisted projections; terminal Turn events carry top-level Turn usage (null when unknown); new Turns start `turn.created`, user `item.added`, `session.in_progress`, `turn.in_progress`; Item events carry `output_index`, null for input Items; assistant text is added in progress with empty content, then an empty part, deltas (a non-streamed final in one delta) and the done events; Session snapshots project the selected MCP credential as retrieve does | H create/reconnect frames; no historical frames in sampled idle interval; J GET streams never ended, terminal `usage` present; SR EVT-09/10 `streams-s1..s4`; MV MV-02 `S1-T1-create-stream` snapshots | C Live; H recorded three-harness disconnect/recovery; T pending actions; J DB order/usage; SR DB sequences and API rendering; MV DB created, in-progress and idle snapshots | Full SSE/Item variants/order (EVT-05..08 deferred); child Turns and Items are not streamed on the Session (U, SAT-09) and are read through the Subagent routes; no replay guarantee or observer-disconnect proof for every state | +| 9 | beta.agents.sessions.list | P: Agent filter, full envelope, cursor paging; unknown keys ignored, limit 0/above 100 clamp; any unresolved cursor, malformed included, is the missing 404; Sessions project the selected MCP credential as retrieve does; failed hosted provisioning projected as on retrieve | S `list-filter.json`, `list-empty-after.json`, `list-owned-cross-filter-cursor.json`, limit/order/unknown-query samples; L SES-10/11/15/16/17; CE ERR-01 `cur-sessions-malformed`, ERR-13/14 | C Live order/cursors/empty/tenant checks; L DB tenant A/B; CE DB cursor matrix tenant A/B; MV DB selected-credential projection; HF DB/HTTP failed projection, tenant B | Core page capacity 100; official cap unknown. Eventual visibility sample is not a required delay | +| 10 | beta.agents.sessions.delete | P: deletion only of a durably idle or failed Session without required actions or pending input; a busy root Turn or pending reservation gives 409 `conflict_error` with no change (subagent child Turns and pending Environment file writes are not checked); owner repeat returns the same 200; owned managed cleanup, user compute retained | S cleanup files 200/deleted; retry-session active cleanup initially 409; Z SES-29 `q5-delete-repeat` 200, SES-30 `q5b-delete-while-in-progress` 409, `q5-delete-never-existed` 404, `q5-delete-while-running` 200 right after an `events.create` 202; HF `038-S2-delete`/`041-S3-delete` 200 after a hosted initialization failure | D recorded real cleanup; C cleanup separately recorded; Z DB matrix D1–D4 with no-write digest, admission lock race and pinned-SDK script; HF HTTP delete of a failed hosted Session | Core returns 409 right after an `events.create` 202 because it admits Turns synchronously (official 200); input awaiting its Environment cannot be cancelled publicly and is unobserved officially; physical purge/retention may end repeat idempotency; caller compute ownership preserved | +| 11 | beta.agents.sessions.events.create | P: 202/empty body, empty-array authenticated no-op, text/cancel/function admission; whitespace-only text is admitted and stored verbatim, empty text/content/input keep the local 400; input the Session cannot accept (result after cancellation, batch while input is pending) and changed results give 409 `conflict_error`/`conflict_error`; in an owned Session an unknown call or a call of another Turn gives 400 `invalid_request_error` without writes; missing/foreign Sessions stay 404; key reuse keeps local 409 `idempotency_conflict` with type `conflict_error`; a Session whose hosted Environment failed to provision gives 409 `conflict_error`/`conflict_error` "the hosted environment failed to provision" | S `second-turn-create.json`, `events-empty/null.json`; H second-input; P SES-04 two whitespace-only messages 202, SES-06/07 empty text/content/input 400; CF EVT-11 `s2-result-unknown-call`/`s2-result-unknown-turn` 400, EVT-12 `s2-result-changed-after-terminal`/`s2-result-after-cancel` 409, EVT-14 duplicate 202, ERR-22 `sessB-message-while-running` 409; HF HI-03 `026-S2-input-after-failure`, `027-S3-input-after-failure` 409 | C Live real continuation/no-op; T qualified message/result/cancel workflows; P DB verbatim whitespace Items and no-write empty-input rejection; CF DB rows CF2–CF4 and CF6–CF9 with tenant B, no-write digest and unchanged pending action, pinned-SDK result script; HF DB/HTTP exact 409, tenant B | Mixed prepared-environment batches, native receipt vs durable acceptance, cancel-before-result-publication timing; unqualified content/tools; empty-input error code/message differ; `["", text]` unobserved officially (SES-08); Claude SDK and MiniMax Code reject whitespace-only messages at admission as a declared native limitation (W6); the official asynchronous pending-input window (ERR-22) is not emulated; Core messages omit call/executor IDs; official order between pending-input and target errors unobserved | +| 12 | beta.agents.sessions.events.stream | P: live-only SSE, typed persisted projections; terminal Turn events carry top-level Turn usage (null when unknown); new Turns start `turn.created`, user `item.added`, `session.in_progress`, `turn.in_progress`; Item events carry `output_index`, null for input Items; assistant text is added in progress with empty content, then an empty part, deltas (a non-streamed final in one delta) and the done events; Session snapshots project the selected MCP credential as retrieve does; a hosted provisioning failure sends `environment.failed` (`environment_error`/`environment_connection_failed`), `error` (`environment_error`/`sandbox_error`, `param` null) and `agent.session.failed`, then GET and creation streams end | H create/reconnect frames; no historical frames in sampled idle interval; J GET streams never ended, terminal `usage` present; SR EVT-09/10 `streams-s1..s4`; MV MV-02 `S1-T1-create-stream` snapshots; HF HI-01/02 `007-S2-events`, `009-S3-events` (stream closed) | C Live; H recorded three-harness disconnect/recovery; T pending actions; J DB order/usage; SR DB sequences and API rendering; MV DB created, in-progress and idle snapshots; HF DB order and HTTP GET stream end | Full SSE/Item variants/order (EVT-05..08 deferred); child Turns and Items are not streamed on the Session (U, SAT-09) and are read through the Subagent routes; no replay guarantee or observer-disconnect proof for every state | | 13 | beta.agents.sessions.turns.retrieve | P: persisted root Turn identity; malformed and child Turn IDs equal missing | H/S contain Turn list payloads; no isolated positive retrieve raw request identified in this set; X SES-28 official `turn_` 404; U SAT-07 `C04` child Turn ID 404 | Recorded H/B scoped Turn recovery (B under the earlier mixed root/child contract); C history uses list; U DB child-ID 404 equal to missing, tenant B | Distinguish list-shape evidence from retrieve wire qualification; full lifecycle/usage; official 404 message text differs | | 14 | beta.agents.sessions.turns.list | P: full envelope, ordered root Turns only; a child Turn, malformed or other unresolved cursor equals a missing one; limit outside 1–100 rejects with the Beta code | S `turns-final/empty-page/limit-high/order-empty.json`; H both directions; L SES-14/15/16/17; X SES-28; U SAT-07 `R06` root-only list beside a child Turn; CE ERR-01 `cur-turns-malformed`, ERR-13 | C Live paging; H/B real child/root identity recorded under the earlier mixed contract; L DB tenant A/B; U DB root-only list and cursor; CE DB cursor matrix tenant A/B | All interleavings, same-timestamp paging, interim/failed usage | | 15 | beta.agents.sessions.items.list | P: scoped root Items, full envelope; limit 0/above 100 clamp within the pinned 1–100 page; a cursor that is not an Item of the Session is 400 ``Invalid session item ID in `after` ``; messages carry `phase`, null for user messages and when no native phase is reported; function results carry `output` and `error`, null when not submitted | S `items-final/empty-page/limit-high.json`; H both directions; L SES-12/13/15/16/17; CE ERR-02 `cur-items-*`; SR SES-25, EVT-09 `s2-items-after-t1`, `s4-items` | C Live; H/T recorded content/coordination/result variants; L DB tenant A/B; CE DB cursor matrix tenant A/B; SR DB/SDK null fields | Full Item union. Newer turn_id filter excluded from pin. Official pages showed a failed result's submitted array output as null; Core keeps the submitted content | diff --git a/services/agents-api/README.md b/services/agents-api/README.md index 70af5e677..9c9eb8e90 100644 --- a/services/agents-api/README.md +++ b/services/agents-api/README.md @@ -281,7 +281,9 @@ ordered setup and [public Environment Templates](../../contracts/agents-api/envi resolve to the same immutable hosted configuration, independently of provider templates. Additional harnesses require separate integration and qualification. Connected describes the authenticated Runtime connection, not native readiness. -Exact hosted failure/expiry semantics remain unverified. +A provisioning failure fails the Session with a safe step and exit-status reason +([initialization failure](../../contracts/agents-api/environment-templates.md#initialization-failure--september-23)); +other exact hosted failure and expiry semantics remain unverified. ## Internal execution device connection From 4d7b4d430d77726e4765c0474b37220a4da74bc2 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 20:49:42 +0000 Subject: [PATCH 05/11] Show the hosted provisioning reason in the Web failure fixture The fixture's failed managed Environment now carries the step and exit status reason that Core reports for a hosted provisioning failure. --- apps/web/e2e/agents-lifecycle.spec.ts | 2 +- apps/web/e2e/fixture-core.mjs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/web/e2e/agents-lifecycle.spec.ts b/apps/web/e2e/agents-lifecycle.spec.ts index 49c89ce2c..1db2e89e9 100644 --- a/apps/web/e2e/agents-lifecycle.spec.ts +++ b/apps/web/e2e/agents-lifecycle.spec.ts @@ -1224,7 +1224,7 @@ test("keeps managed Environment resource and terminal event states fail-closed", }); await page.reload(); await expect(page.getByText("Session cannot continue", { exact: true })).toBeVisible(); - await expect(page.getByText("The environment is no longer available for this input.", { exact: true })).toBeVisible(); + await expect(page.getByText('Failed to provision environment: script "setup_commands[0]" failed with exit code 3', { exact: true })).toBeVisible(); await expect(page.getByLabel("Message the Agent")).toBeDisabled(); opened = await openEnvironmentDialog(page); await expect(opened.panel).toContainText("Managed Environment failed"); diff --git a/apps/web/e2e/fixture-core.mjs b/apps/web/e2e/fixture-core.mjs index 32f64aaae..024a6a76f 100644 --- a/apps/web/e2e/fixture-core.mjs +++ b/apps/web/e2e/fixture-core.mjs @@ -511,7 +511,8 @@ function applyEnvironmentScenario(value) { skills: [], }; session.status = "failed"; - session.error = "The environment is no longer available for this input."; + // A hosted provisioning failure reports the failed step and exit status only. + session.error = 'Failed to provision environment: script "setup_commands[0]" failed with exit code 3'; session.required_actions = []; return; } From 37cbd074cfc96c8959bf642e7e6ab1c210d857b4 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:17:43 +0000 Subject: [PATCH 06/11] Renumber the Environment failure migration after main's 000061 Main added 000061_project_api_keys.sql; the Environment failure columns move to 000062 unchanged. --- contracts/agents-api/history-events-usage.md | 2 +- contracts/agents-api/official-semantics-alignment.md | 2 +- ...1_environment_failure.sql => 000062_environment_failure.sql} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename services/agents-api/migrations/{000061_environment_failure.sql => 000062_environment_failure.sql} (100%) diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index 6bb2a2b66..cdaf726b5 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -370,5 +370,5 @@ Evidence: campaign scan 6 HI-01..04 (private nullable `param` on stream errors. Core Web renders the failed Session and its error from the snapshot and ignores the error event. -Environments that failed before migration `000061` have no recorded reason; they +Environments that failed before migration `000062` have no recorded reason; they keep their earlier projection and events. diff --git a/contracts/agents-api/official-semantics-alignment.md b/contracts/agents-api/official-semantics-alignment.md index 1e98d8387..e4450e5a3 100644 --- a/contracts/agents-api/official-semantics-alignment.md +++ b/contracts/agents-api/official-semantics-alignment.md @@ -762,7 +762,7 @@ Decisions: from 1 to 255. The Store composes the reason from a fixed step label and integers, so commands, env values, package names, paths and process output cannot reach the reason, events, logs or responses. -- **Storage.** The additive migration `000061_environment_failure.sql` adds the +- **Storage.** The additive migration `000062_environment_failure.sql` adds the nullable `environments.failure_reason` and `failed_at`; a check ties them to `status = failed` and bounds the reason to 256 characters. Environments that failed earlier keep NULL and their previous projection and events; new input diff --git a/services/agents-api/migrations/000061_environment_failure.sql b/services/agents-api/migrations/000062_environment_failure.sql similarity index 100% rename from services/agents-api/migrations/000061_environment_failure.sql rename to services/agents-api/migrations/000062_environment_failure.sql From 4a91638ee434d8a5ff4cff1f21384ef2a47562cd Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:18:20 +0000 Subject: [PATCH 07/11] Refuse to roll back recorded hosted provisioning failures The 000062 Down now locks environments and raises when any failure reason is recorded, following the repository's rollback guards. A migration test covers the additive upgrade, the column checks and both rollback outcomes. --- .../environment_failure_migration_test.go | 85 +++++++++++++++++++ .../migrations/000062_environment_failure.sql | 9 ++ 2 files changed, 94 insertions(+) create mode 100644 services/agents-api/internal/store/environment_failure_migration_test.go diff --git a/services/agents-api/internal/store/environment_failure_migration_test.go b/services/agents-api/internal/store/environment_failure_migration_test.go new file mode 100644 index 000000000..20294c2d9 --- /dev/null +++ b/services/agents-api/internal/store/environment_failure_migration_test.go @@ -0,0 +1,85 @@ +package store + +import ( + "context" + "database/sql" + "os" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +// The failure columns are additive for existing Environments, and a rollback +// refuses to discard a recorded hosted provisioning failure. +func TestEnvironmentFailureMigrationGuardsRecordedFailures(t *testing.T) { + _, pool := testStore(t) + ctx := t.Context() + schema := "environment_failure_" + uuid.NewString()[:8] + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := pool.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if _, err := pool.Exec(context.Background(), "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error(err) + } + }) + cfg := pool.Config().ConnConfig.Copy() + cfg.RuntimeParams["search_path"] = schema + db := sql.OpenDB(stdlib.GetConnector(*cfg)) + t.Cleanup(func() { _ = db.Close() }) + provider, err := goose.NewProvider(goose.DialectPostgres, db, os.DirFS("../../migrations"), goose.WithTableName("agents_api_schema_version")) + if err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 61); err != nil { + t.Fatal(err) + } + session, environment := uuid.NewString(), uuid.NewString() + if _, err := db.ExecContext(ctx, `INSERT INTO sessions(id,tenant_id,engine,idempotency_key,request_hash,configuration) + VALUES ($1,$2,'codex','historical','historical','{"environment":{"type":"openai_hosted"}}')`, session, uuid.NewString()); err != nil { + t.Fatal(err) + } + // A failure recorded before this migration keeps no reason. + if _, err := db.ExecContext(ctx, "INSERT INTO environments(id,session_id,status) VALUES ($1,$2,'failed')", environment, session); err != nil { + t.Fatal(err) + } + upgrade := func() { + t.Helper() + if _, err := provider.UpTo(ctx, 62); err != nil { + t.Fatal(err) + } + var reason, failedAt sql.NullString + if err := db.QueryRowContext(ctx, "SELECT failure_reason, failed_at::text FROM environments WHERE id=$1", environment).Scan(&reason, &failedAt); err != nil || reason.Valid || failedAt.Valid { + t.Fatal("migration invented a failure reason", reason, failedAt, err) + } + } + upgrade() + for _, statement := range []string{ + "UPDATE environments SET failure_reason='reason' WHERE id=$1", + "UPDATE environments SET status='connected', failure_reason='reason', failed_at=clock_timestamp() WHERE id=$1", + "UPDATE environments SET failure_reason=repeat('x', 257), failed_at=clock_timestamp() WHERE id=$1", + } { + if _, err := db.ExecContext(ctx, statement, environment); err == nil || !strings.Contains(err.Error(), "environment_failure_recorded") { + t.Fatal("inconsistent failure accepted", statement, err) + } + } + if _, err := provider.DownTo(ctx, 61); err != nil { + t.Fatal("rollback without recorded failures failed", err) + } + upgrade() + if _, err := db.ExecContext(ctx, "UPDATE environments SET failure_reason='Failed to provision environment', failed_at=clock_timestamp() WHERE id=$1", environment); err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(ctx, 61); err == nil || !strings.Contains(err.Error(), "Cannot remove recorded hosted provisioning failures") { + t.Fatal("rollback discarded a recorded failure", err) + } + var reason string + if err := db.QueryRowContext(ctx, "SELECT failure_reason FROM environments WHERE id=$1", environment).Scan(&reason); err != nil || reason != "Failed to provision environment" { + t.Fatal("refused rollback changed the failure", reason, err) + } +} diff --git a/services/agents-api/migrations/000062_environment_failure.sql b/services/agents-api/migrations/000062_environment_failure.sql index cda1a0d80..6a2f3861b 100644 --- a/services/agents-api/migrations/000062_environment_failure.sql +++ b/services/agents-api/migrations/000062_environment_failure.sql @@ -12,6 +12,15 @@ ALTER TABLE environments ); -- +goose Down +LOCK TABLE environments IN ACCESS EXCLUSIVE MODE; +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM environments WHERE failure_reason IS NOT NULL) THEN + RAISE EXCEPTION 'Cannot remove recorded hosted provisioning failures'; + END IF; +END $$; +-- +goose StatementEnd ALTER TABLE environments DROP CONSTRAINT environment_failure_recorded, DROP COLUMN failed_at, From fb39526974f2a62fef6b6c7440499c416fef9a10 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:19:46 +0000 Subject: [PATCH 08/11] Keep Core's stream_interrupted frame without param Released clients validate exactly code, type and message on stream errors and would report the interruption as an invalid stream. Core's own interruption frame keeps that shape; the official-shaped hosted failure error event keeps param null. Both shapes are tested. --- contracts/agents-api/history-events-usage.md | 6 +++--- .../agents-api/official-semantics-alignment.md | 4 +++- contracts/agents-api/openapi.yaml | 5 +++-- contracts/agents-api/v1/events.go | 5 +++-- packages/agents-client/src/types.ts | 2 +- .../internal/api/hosted_failure_test.go | 17 +++++++++++++++++ services/agents-api/internal/api/stream.go | 11 +++++++++-- 7 files changed, 39 insertions(+), 11 deletions(-) diff --git a/contracts/agents-api/history-events-usage.md b/contracts/agents-api/history-events-usage.md index cdaf726b5..774c02299 100644 --- a/contracts/agents-api/history-events-usage.md +++ b/contracts/agents-api/history-events-usage.md @@ -353,9 +353,9 @@ Evidence: campaign scan 6 HI-01..04 (private - **Payloads.** `environment.error` is `{type: environment_error, code: environment_connection_failed, message: "The environment failed to connect."}`. The `error` event carries the pinned `SessionError`: `{type: environment_error, - code: sandbox_error, message: , param: null}`. Every `error` event - now includes `param` (null when unset), including Core's own - `stream_interrupted`. The `agent.session.failed` snapshot has `status: failed`, + code: sandbox_error, message: , param: null}`. Core's own + `stream_interrupted` frame keeps its three-field error without `param`, which + released clients validate exactly. The `agent.session.failed` snapshot has `status: failed`, the reason as `error`, `required_actions: []` and the failure time as `last_active_at`, identical to later retrieve and list reads. Pending input settled by the failure is captured in the same snapshot. diff --git a/contracts/agents-api/official-semantics-alignment.md b/contracts/agents-api/official-semantics-alignment.md index e4450e5a3..538524155 100644 --- a/contracts/agents-api/official-semantics-alignment.md +++ b/contracts/agents-api/official-semantics-alignment.md @@ -773,7 +773,9 @@ Decisions: - **Scope of the terminal state.** Only a recorded hosted provisioning failure makes the Session terminal. A Turn failure still leaves GET streams open, and a GET stream opened after the failure stays open; that case was not observed. -- **Clients.** Every `error` event now carries `param`. The TypeScript client +- **Clients.** Official-shaped `error` events carry `param: null`; Core's own + `stream_interrupted` frame keeps its three-field error without `param`, so + released clients still report it as an interruption. The TypeScript client delivers error events other than Core's `stream_interrupted` to `onEvent` before the failed snapshot, instead of raising them. Core Web already renders the failed Session, its error and the blocked input. diff --git a/contracts/agents-api/openapi.yaml b/contracts/agents-api/openapi.yaml index 1b2af9c62..c51455a40 100644 --- a/contracts/agents-api/openapi.yaml +++ b/contracts/agents-api/openapi.yaml @@ -2483,8 +2483,9 @@ definitions: type: string param: description: |- - Param is the pinned SessionError field. A top-level error event always - carries it, null when unset; Environment state errors omit it. + Param is the pinned SessionError field. An error SessionEvent always + carries it, null when unset; Environment state errors and Core's own + stream_interrupted frame omit it. type: string x-nullable: true type: diff --git a/contracts/agents-api/v1/events.go b/contracts/agents-api/v1/events.go index 3f6f41b40..d344623d0 100644 --- a/contracts/agents-api/v1/events.go +++ b/contracts/agents-api/v1/events.go @@ -29,8 +29,9 @@ type StreamError struct { Code string `json:"code"` Type string `json:"type"` Message string `json:"message"` - // Param is the pinned SessionError field. A top-level error event always - // carries it, null when unset; Environment state errors omit it. + // Param is the pinned SessionError field. An error SessionEvent always + // carries it, null when unset; Environment state errors and Core's own + // stream_interrupted frame omit it. Param *string `json:"param,omitempty" extensions:"x-nullable"` } diff --git a/packages/agents-client/src/types.ts b/packages/agents-client/src/types.ts index 3083d1891..f09f65ee2 100644 --- a/packages/agents-client/src/types.ts +++ b/packages/agents-client/src/types.ts @@ -629,7 +629,7 @@ export interface StreamError { code: string; type: string; message: string; - /** Present on error events (null when unset); Environment state errors omit it. */ + /** Present on Session error events (null when unset); Environment state errors and older or interruption frames omit it. */ param?: string | null; } diff --git a/services/agents-api/internal/api/hosted_failure_test.go b/services/agents-api/internal/api/hosted_failure_test.go index ddc351005..433217379 100644 --- a/services/agents-api/internal/api/hosted_failure_test.go +++ b/services/agents-api/internal/api/hosted_failure_test.go @@ -192,3 +192,20 @@ func TestHostedProvisioningFailureInputConflict(t *testing.T) { t.Fatal("internal callers no longer see an unavailable Environment") } } + +// Core's own interruption frame keeps the three-field error that released +// clients validate exactly; official error events carry param null. +func TestStreamInterruptionFrameOmitsParam(t *testing.T) { + var frame []byte + writeStreamFailure(func(data []byte) error { frame = data; return nil }, "session") + name, data, ok := strings.Cut(strings.TrimSuffix(string(frame), "\n\n"), "\n") + var event map[string]any + if !ok || name != "event: error" || json.Unmarshal([]byte(strings.TrimPrefix(data, "data: ")), &event) != nil { + t.Fatalf("frame %q", frame) + } + want := map[string]any{"type": "error", "event_id": event["event_id"], "session_id": "session", "error": map[string]any{ + "code": "stream_interrupted", "type": "server_error", "message": "The live stream was interrupted. Reconnect and retrieve the Session and its saved Items to recover."}} + if id, _ := event["event_id"].(string); id == "" || !reflect.DeepEqual(event, want) { + t.Fatalf("interruption frame %s", data) + } +} diff --git a/services/agents-api/internal/api/stream.go b/services/agents-api/internal/api/stream.go index aeda62815..f4923f311 100644 --- a/services/agents-api/internal/api/stream.go +++ b/services/agents-api/internal/api/stream.go @@ -241,9 +241,16 @@ func withTurnUsage(event v1.SessionEvent) v1.SessionEvent { return event } +// writeStreamFailure sends Core's own interruption frame. Unlike official error +// events it omits param: released clients validate exactly code, type and message. func writeStreamFailure(write func([]byte) error, session string) { - event := v1.SessionEvent{Type: "error", EventID: uuid.NewString(), SessionID: session, - Error: &v1.StreamError{Code: "stream_interrupted", Type: "server_error", Message: "The live stream was interrupted. Reconnect and retrieve the Session and its saved Items to recover."}} + event := struct { + Type string `json:"type"` + EventID string `json:"event_id"` + SessionID string `json:"session_id"` + Error v1.StreamError `json:"error"` + }{"error", uuid.NewString(), session, + v1.StreamError{Code: "stream_interrupted", Type: "server_error", Message: "The live stream was interrupted. Reconnect and retrieve the Session and its saved Items to recover."}} payload, _ := json.Marshal(event) _ = write([]byte(fmt.Sprintf("event: error\ndata: %s\n\n", payload))) } From 9319da82764877149c3a27a1f521085323a89b14 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:20:04 +0000 Subject: [PATCH 09/11] Document the GET event stream end after a hosted provisioning failure The contracts README and CONTRIBUTING said GET streams never end on their own; they now name the terminal hosted failure and Session deletion as the only server-side ends. --- CONTRIBUTING.md | 5 ++++- contracts/agents-api/README.md | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 14f04ff33..58065ae06 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2554,7 +2554,10 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti connection comment and ends at once, admitting nothing and following no work, because official same-key requests create distinct Sessions. Retry the same request/key with `stream=false`, or use the GET events stream, to recover. GET - event streams keep their live-only start and never end on settlement. + event streams keep their live-only start and never end on settlement or a Turn + failure; the only server-side end is the terminal `agent.session.failed` of a + hosted provisioning failure (and Session deletion), since that Session can + never run again. Disconnect never cancels admitted work. Official observations cover `none` creation; self-hosted, hosted and no-input stream lifetimes and the retry behavior are local choices, and the separate SDK one-Turn helper does not diff --git a/contracts/agents-api/README.md b/contracts/agents-api/README.md index f77757cfd..d1950e825 100644 --- a/contracts/agents-api/README.md +++ b/contracts/agents-api/README.md @@ -821,7 +821,10 @@ reservation expires or fails. A creation that admitted nothing ends right after `created`. A settlement that records no event ends the stream after the events up to the cursor read with a settled projection in one snapshot; another client's work drained before that read can still be sent. Observe later Turns with the GET -event stream, which never ends on its own. Terminal Turn events carry the Turn +event stream, which does not end on settlement or a Turn failure; it ends only +after the terminal `agent.session.failed` of a hosted provisioning failure, as +officially observed ([initialization failure](environment-templates.md#initialization-failure--september-23)), +or when the Session is deleted. Terminal Turn events carry the Turn snapshot's `usage` at the top level, null when unknown. The local `Idempotency-Key` creation extension shares identity across response From 69535fc0976f6abe9d113baae1c46b08208968e1 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:20:41 +0000 Subject: [PATCH 10/11] Describe the lenient initialization receipt decoding precisely The docs called the failed receipt strict. The decoder deliberately ignores other fields so older images keep working; the only value taken from it is an integer exit_code from 1 to 255, and output fields are never read. --- contracts/agents-api/environment-templates.md | 14 +++++++++----- .../agents-api/official-semantics-alignment.md | 6 ++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/contracts/agents-api/environment-templates.md b/contracts/agents-api/environment-templates.md index 3b5bdf402..313954f5b 100644 --- a/contracts/agents-api/environment-templates.md +++ b/contracts/agents-api/environment-templates.md @@ -458,11 +458,15 @@ The reason names only the failed step and its exit status: "Anything else" covers timeouts, the thirty-minute budget, unknown effects, missing or malformed receipts, receipts without `exit_code` from Runtime images built before this change, Plugin and capability installation, bootstrap -rejection and Core restart during initialization. Core accepts an exit status -only from a strict version-1 failed receipt with process status 1 and no stderr, -as an integer from 1 to 255, and the Store composes the reason from a fixed label -and integers. Commands, env values, package names, paths and any process output -therefore never reach the reason, events, logs or responses. The failed step is +rejection and Core restart during initialization. Core treats a step as +confirmed failed only when the process exits 1 with empty stderr and stdout +decodes as a version-1 receipt whose `outcome` is `failed`. The decoder is +deliberately lenient so older images keep working: other receipt fields are +ignored and never read. The only value ever taken from the receipt is +`exit_code`, used only when it is an integer from 1 to 255. The Store composes the +reason from a fixed label and integers, so commands, env values, package names, +paths and any process output never reach the reason, events, logs or responses. +The failed step is not retried and later steps do not run. ## System packages diff --git a/contracts/agents-api/official-semantics-alignment.md b/contracts/agents-api/official-semantics-alignment.md index 538524155..4edf6d489 100644 --- a/contracts/agents-api/official-semantics-alignment.md +++ b/contracts/agents-api/official-semantics-alignment.md @@ -757,8 +757,10 @@ Decisions: - **No output.** The shared Runtime initializer adds only an integer `exit_code` to its failed receipt, and only for a step run inside its bwrap isolation; Runtime helpers, signals and other errors keep the generic receipt, and the - exception is never serialized. Core accepts the status only from a strict - version-1 failed receipt with process status 1 and no stderr, as an integer + exception is never serialized. Core confirms a failed step only when the + process exits 1 with empty stderr and a version-1 `failed` receipt. The + decoder is deliberately lenient for older images and ignores other fields; the + only value ever taken from the receipt is `exit_code`, and only as an integer from 1 to 255. The Store composes the reason from a fixed step label and integers, so commands, env values, package names, paths and process output cannot reach the reason, events, logs or responses. From 3ed67b9f7a00cc850791a7602bc27a8fe4e0326f Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Wed, 23 Sep 2026 21:23:49 +0000 Subject: [PATCH 11/11] Test receipt outcomes, the Skill label and output leakage runRuntimeSetup and installInitialFile tests cover stderr, other process statuses, exit_code 0, 256, negative, null and mistyped, ignored and duplicate fields, raw output and a Plugin step. The store tests add the Skill label end to end, and the fake Provider now returns canary output in receipt fields; the HTTP test also scans captured logs for it. --- .../internal/execution/runtime_setup_test.go | 135 ++++++++++++++++++ ...sted_initialization_failure_public_test.go | 69 +++++++-- 2 files changed, 196 insertions(+), 8 deletions(-) create mode 100644 services/agents-api/internal/execution/runtime_setup_test.go diff --git a/services/agents-api/internal/execution/runtime_setup_test.go b/services/agents-api/internal/execution/runtime_setup_test.go new file mode 100644 index 000000000..b64d0a0aa --- /dev/null +++ b/services/agents-api/internal/execution/runtime_setup_test.go @@ -0,0 +1,135 @@ +package execution + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentcapabilities" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/sandbox" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +const setupCanary = "CANARY-runtime-setup-9b3e" + +// receiptProvider answers every initialization command with one fixed result. +type receiptProvider struct { + sandbox.Provider + result sandbox.CommandResult + err error +} + +func (p receiptProvider) RunCommand(context.Context, sandbox.Reference, sandbox.Command) (sandbox.CommandResult, error) { + return p.result, p.err +} + +// Only a process status 1, empty stderr and a version-1 failed receipt confirm a +// failed step, and only an integer exit_code from 1 to 255 is taken from it. +// Other receipt fields are ignored, never read; everything else stays generic. +func TestRuntimeSetupReceiptOutcomes(t *testing.T) { + setup := runtimeSetupOperation{Version: 1, Action: "setup", Network: "enabled", Command: "echo " + setupCanary, CWD: "/workspace", Index: 2} + plugin := runtimeSetupOperation{Capabilities: &agentcapabilities.Operation{Version: 1, Action: "plugin"}} + unconfirmed := -1 + for _, test := range []struct { + name string + operation runtimeSetupOperation + result sandbox.CommandResult + err error + exitCode int // -1 unconfirmed, 0 confirmed without a status + }{ + {"completed", setup, sandbox.CommandResult{Stdout: `{"version":1,"outcome":"completed"}`}, nil, -2}, + {"exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}, nil, 3}, + {"highest exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":255}` + "\n"}, nil, 255}, + {"old receipt", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed"}`}, nil, 0}, + {"zero exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":0}`}, nil, 0}, + {"exit status above 255", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":256}`}, nil, 0}, + {"negative exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":-9}`}, nil, 0}, + {"null exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":null}`}, nil, 0}, + // Lenient for older and newer images: other fields are never read. + {"ignored output field", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3,"output":"` + setupCanary + `"}`}, nil, 3}, + {"duplicate key keeps the last", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3,"exit_code":4}`}, nil, 4}, + {"string exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":"3"}`}, nil, unconfirmed}, + {"fractional exit status", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3.5}`}, nil, unconfirmed}, + {"stderr", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`, Stderr: setupCanary}, nil, unconfirmed}, + {"process status 2", setup, sandbox.CommandResult{ExitCode: 2, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}, nil, unconfirmed}, + {"process status 0", setup, sandbox.CommandResult{Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}, nil, unconfirmed}, + {"completed with status 1", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"completed"}`}, nil, unconfirmed}, + {"version 2", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":2,"outcome":"failed","exit_code":3}`}, nil, unconfirmed}, + {"raw output", setup, sandbox.CommandResult{ExitCode: 3, Stdout: setupCanary}, nil, unconfirmed}, + {"trailing output", setup, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}` + setupCanary}, nil, unconfirmed}, + {"provider error", setup, sandbox.CommandResult{}, sandbox.ErrCommandUnconfirmed, unconfirmed}, + {"plugin step", plugin, sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}, nil, unconfirmed}, + } { + t.Run(test.name, func(t *testing.T) { + err := runRuntimeSetup(t.Context(), receiptProvider{result: test.result, err: test.err}, sandbox.Reference{}, test.operation) + var failed *runtimeStepFailure + switch { + case test.exitCode == -2: + if err != nil { + t.Fatal(err) + } + case test.exitCode == unconfirmed: + if err == nil || errors.As(err, &failed) { + t.Fatal("unconfirmed result became a confirmed failure", err) + } + default: + if !errors.As(err, &failed) || failed.exitCode != test.exitCode { + t.Fatal("confirmed failure", err, failed) + } + } + if err != nil && strings.Contains(err.Error(), setupCanary) { + t.Fatal("receipt or output reached the error", err) + } + }) + } +} + +func TestRuntimeSetupFailureLabels(t *testing.T) { + for operation, want := range map[string]store.ProvisioningFailure{ + "setup": {Step: store.ProvisioningSetupCommand, Index: 2, ExitCode: 3}, + "python": {Step: store.ProvisioningPythonPackages, Index: 2, ExitCode: 3}, + "npm": {Step: store.ProvisioningNPMPackages, Index: 2, ExitCode: 3}, + "system": {Step: store.ProvisioningSystemPackages, Index: 2, ExitCode: 3}, + "skill": {Step: store.ProvisioningSkill, Index: 2, ExitCode: 3}, + "configure": {}, + "": {}, + } { + if got := (runtimeSetupOperation{Action: operation, Index: 2}).provisioningFailure(3); got != want { + t.Errorf("%q: %+v", operation, got) + } + } + // setupOperations numbers setup commands from zero. + operations := setupOperations(store.EnvironmentSetup{Commands: []store.SetupCommand{{Command: "a"}, {Command: "b"}}}) + if len(operations) != 3 || operations[1].Index != 0 || operations[2].Index != 1 || operations[2].Action != "setup" { + t.Fatal("setup command positions", operations) + } +} + +// The file writer exits 0 with a failed receipt when it committed nothing; an +// unknown outcome, stderr or a wrong size stays unconfirmed. +func TestInitialFileReceiptOutcomes(t *testing.T) { + body := []byte(setupCanary) + file := store.InitialFileMetadata{Path: "/workspace/a"} + size := int64(len(body)) + file.SizeBytes = &size + for _, test := range []struct { + name string + result sandbox.CommandResult + confirmed bool + }{ + {"failed", sandbox.CommandResult{Stdout: `{"version":1,"outcome":"failed","error":"write_failed"}`}, true}, + {"unknown", sandbox.CommandResult{Stdout: `{"version":1,"outcome":"unknown","error":"write_failed"}`}, false}, + {"stderr", sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed"}`, Stderr: setupCanary}, false}, + {"wrong size", sandbox.CommandResult{Stdout: `{"version":1,"outcome":"completed","size_bytes":1}`}, false}, + {"raw output", sandbox.CommandResult{ExitCode: 2, Stdout: setupCanary}, false}, + } { + t.Run(test.name, func(t *testing.T) { + err := installInitialFile(t.Context(), receiptProvider{result: test.result}, sandbox.Reference{}, file, body) + var failed *runtimeStepFailure + if err == nil || errors.As(err, &failed) != test.confirmed || strings.Contains(err.Error(), setupCanary) { + t.Fatal(err) + } + }) + } +} diff --git a/services/agents-api/internal/store/hosted_initialization_failure_public_test.go b/services/agents-api/internal/store/hosted_initialization_failure_public_test.go index ff5fe8164..c871a089d 100644 --- a/services/agents-api/internal/store/hosted_initialization_failure_public_test.go +++ b/services/agents-api/internal/store/hosted_initialization_failure_public_test.go @@ -1,17 +1,20 @@ package store_test import ( + "archive/zip" "bytes" "context" "encoding/json" "errors" "fmt" "io" + "log/slog" "net/http" "net/http/httptest" "reflect" "strconv" "strings" + "sync" "testing" "time" @@ -26,6 +29,29 @@ import ( const hostedFailureCanary = "CANARY-hosted-init-7c21" +// leakyReceipt is a failed receipt that also carries canary output in fields +// Core must never read, as a leaking or newer Runtime could send. +func leakyReceipt(fields string) sandbox.CommandResult { + return sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed",` + fields + `"output":"` + hostedFailureCanary + `","stderr":"` + hostedFailureCanary + `"}`} +} + +func hostedFailureSkill(t *testing.T) store.EnvironmentSkill { + t.Helper() + var archive bytes.Buffer + writer := zip.NewWriter(&archive) + file, err := writer.CreateHeader(&zip.FileHeader{Name: "proof/SKILL.md", Method: zip.Store}) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write([]byte("---\nname: proof\ndescription: A proof.\n---\n" + hostedFailureCanary)); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return store.EnvironmentSkill{Metadata: store.EnvironmentSkillMetadata{Type: "inline", Name: "proof", Description: "A proof."}, Archive: archive.Bytes()} +} + // hostedFailureProvider fails one initialization step with a controlled result. // Every failure it reports is produced next to canary output, which the // initializer discards and Core must never publish. @@ -110,7 +136,6 @@ func failHostedInitialization(t *testing.T, s *store.Store, tenant string, envir // with the safe reason and agent.session.failed; reads and events agree, and a // confirmed step names only its label and exit status. func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { - failed := func(stdout string) sandbox.CommandResult { return sandbox.CommandResult{ExitCode: 1, Stdout: stdout} } commands := []store.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 0"}, {Command: "echo " + hostedFailureCanary + "; exit 3"}, {Command: "touch never"}} type failure struct { fail string @@ -126,16 +151,16 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { steps []string }{ {"setup exit status", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, - failure{fail: "setup", result: failed(`{"version":1,"outcome":"failed","exit_code":3}`)}, + failure{fail: "setup", result: leakyReceipt(`"exit_code":3,`)}, `Failed to provision environment: script "setup_commands[0]" failed with exit code 3`, []string{"configure", "setup"}}, {"later setup command", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands}}, - failure{fail: "setup", skip: 1, result: failed(`{"version":1,"outcome":"failed","exit_code":3}` + "\n")}, + failure{fail: "setup", skip: 1, result: sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}` + "\n"}}, `Failed to provision environment: script "setup_commands[1]" failed with exit code 3`, []string{"configure", "setup", "setup"}}, {"python package", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Packages: v1.EnvironmentPackages{Python: []string{"parsar-nonexistent-zz"}}, Commands: commands[2:]}}, - failure{fail: "python", result: failed(`{"version":1,"outcome":"failed","exit_code":1}`)}, + failure{fail: "python", result: leakyReceipt(`"exit_code":1,`)}, `Failed to provision environment: script "Python package installation" failed with exit code 1`, []string{"configure", "python"}}, {"old image without exit status", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, - failure{fail: "setup", result: failed(`{"version":1,"outcome":"failed"}`)}, + failure{fail: "setup", result: leakyReceipt("")}, "Failed to provision environment: initialization did not complete", []string{"configure", "setup"}}, {"unknown effect", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Commands: commands[1:]}}, failure{fail: "setup", err: sandbox.ErrCommandUnconfirmed}, @@ -144,8 +169,11 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { failure{fail: "setup", result: sandbox.CommandResult{ExitCode: 3, Stdout: hostedFailureCanary, Stderr: hostedFailureCanary}}, "Failed to provision environment: initialization did not complete", []string{"configure", "setup"}}, {"initial file", store.CreateSessionInput{InitialFiles: []store.InitialFile{{Type: "inline", Path: "/workspace/a", Data: []byte(hostedFailureCanary)}}}, - failure{fail: "file", result: sandbox.CommandResult{Stdout: `{"version":1,"outcome":"failed","error":"write_failed"}`}}, + failure{fail: "file", result: sandbox.CommandResult{Stdout: `{"version":1,"outcome":"failed","error":"write_failed","detail":"` + hostedFailureCanary + `"}`}}, "Failed to provision environment: initial file installation failed", []string{"file"}}, + {"Skill", store.CreateSessionInput{Initialization: store.EnvironmentSetup{Skills: []store.EnvironmentSkill{hostedFailureSkill(t)}, Commands: commands[2:]}}, + failure{fail: "skill", result: leakyReceipt("")}, + "Failed to provision environment: Skill installation failed", []string{"configure", "skill"}}, } { t.Run(test.name, func(t *testing.T) { s := hostedFailureStore(t) @@ -249,8 +277,12 @@ func TestHostedInitializationFailurePublicHTTP(t *testing.T) { Metadata: map[string]string{"case": "setup-exit3"}, }) key := uuid.NewString() - p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", - result: sandbox.CommandResult{ExitCode: 1, Stdout: `{"version":1,"outcome":"failed","exit_code":3}`}} + // The failed receipt carries canary output in fields Core must never read. + p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", result: leakyReceipt(`"exit_code":3,`)} + logs := &lockedBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previous) }) w, _ := managedWorker(t, s, key, p) auth, err := api.NewAuthenticator([]api.APIKey{ {OrganizationID: "test-org", ProjectID: tenant, SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: device.HashCredential(token), TenantID: tenant}, @@ -381,4 +413,25 @@ func TestHostedInitializationFailurePublicHTTP(t *testing.T) { t.Fatal("initialization output reached a response", body) } } + // Logs were captured (the failed step is logged) and hold no output either. + if logged := logs.String(); !strings.Contains(logged, "managed Runtime file initialization incomplete") || strings.Contains(logged, hostedFailureCanary) { + t.Fatal("log capture", logged) + } +} + +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(data) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.String() }