diff --git a/AGENTS.md b/AGENTS.md index be5044f34..500d502b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ Do not multiply entities without necessity. The long-term goal is minimal code, - A new Sandbox Provider, Harness, model provider or vendor feature changes only its adapter. It adds no Core execution path, store table or column, migration, deployment or configuration field, API field or Web UI specific to one vendor or Harness. The [Sandbox Provider guide](docs/sandbox-provider.md) and [Harness onboarding](contracts/agents-api/harness-onboarding.md) describe how to add an adapter. - When the protocol cannot express what an adapter needs, change the protocol. Never add an optional side interface for one implementation. -- Implementing the declared `CheckpointProvider` lifecycle in one vendor's Provider is an adapter change. A vendor-only pause interface, a Core path for that vendor, vendor receipts in the store or a vendor idle setting in the deployment is not. +- Implementing the declared checkpoint lifecycle in one vendor's Provider is an adapter change. A vendor-only pause interface, a Core path for that vendor, vendor receipts in the store or a vendor idle setting in the deployment is not. - Fix shared lifecycle, admission, cancellation, reuse and performance problems in the common flow, never in a branch selected by Harness, Runtime or vendor name. Core preparation and execution never branch on operating system or Environment source; platform support requires native CI builds and automated tests. - Each Harness runs its own model and tool loop through a maintained upstream SDK or native protocol, in the Environment's declared workspace directory. Its native history or configuration directory is never the workspace. Never build a second executor, a hand-written model/tool loop or a compatibility framework to fabricate parity. The public API and persistence never depend on one engine's native item types. diff --git a/apps/web/src/features/sandbox/NodeList.test.ts b/apps/web/src/features/sandbox/NodeList.test.ts index d32fab58e..d90258500 100644 --- a/apps/web/src/features/sandbox/NodeList.test.ts +++ b/apps/web/src/features/sandbox/NodeList.test.ts @@ -21,8 +21,6 @@ describe("node state", () => { expect(nodeState(node("a", { online: false }), [], true, core)).toBe("unconfirmed"); expect(nodeState(node("a", { core_url: "https://core-old.example" }), [], false, core)).toBe("old_address"); expect(nodeState(node("a", { online: false, core_url: "https://core-old.example" }), [], false, core)).toBe("old_address"); - // A node Core did not enroll, such as a file-managed local one, reports no address: unknown, not old. - expect(nodeState(node("a", { core_url: "" }), [], false, core)).toBe("available"); expect(nodeState(node("a", { online: false, cleanup_pending: 2 }), [], false, core)).toBe("offline"); expect(nodeState(node("a", { provider_ready: false }), [], false, core)).toBe("degraded"); }); diff --git a/apps/web/src/features/sandbox/NodeList.tsx b/apps/web/src/features/sandbox/NodeList.tsx index aa8bb7d50..b5c4fd979 100644 --- a/apps/web/src/features/sandbox/NodeList.tsx +++ b/apps/web/src/features/sandbox/NodeList.tsx @@ -15,11 +15,10 @@ export type NodeState = "unconfirmed" | "old_address" | "offline" | "degraded" | /** * Whether a node enrolled with another Core address than the deployment's - * `coreUrl`. An empty address is unknown, not old: a node Core did not enroll, - * such as a file-managed local one, reports none. + * `coreUrl`. While the deployment is unknown, no node is on an old address. */ export function onOldAddress(node: SandboxNode, coreUrl: string): boolean { - return Boolean(node.core_url && coreUrl && node.core_url !== coreUrl); + return coreUrl !== "" && node.core_url !== coreUrl; } /** diff --git a/apps/web/src/lib/locale-strings.ts b/apps/web/src/lib/locale-strings.ts index 880588c9c..e6080a26d 100644 --- a/apps/web/src/lib/locale-strings.ts +++ b/apps/web/src/lib/locale-strings.ts @@ -203,7 +203,6 @@ export const chinese = { "Couldn't confirm the sandbox change": "沙箱更改未能确认", "Sandbox state couldn't be read": "无法读取沙箱状态", "Nodes": "节点", - "Local nodes run on the Core server. Capacity and counts are reported by Core.": "本地节点运行在 Core 服务器上。容量和资源数量由 Core 上报。", "Sandbox nodes": "沙箱节点", "Node": "节点", "Health": "健康状态", @@ -244,7 +243,6 @@ export const chinese = { "Automatic placement": "自动分配", "available": "可用", "unavailable": "不可用", - "Optional. Local nodes run on the Core server. A selected node must be available; Core will not fall back to another node.": "可选。本地节点运行在 Core 服务器上。所选节点必须可用；Core 不会自动改用其他节点。", "Loading nodes…": "正在加载节点…", "Retry directory": "重新加载节点目录", "No nodes are registered.": "尚未注册节点。", diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index c04c1097c..e26119271 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -6222,7 +6222,7 @@ paths: post: consumes: - application/json - description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. + description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. parameters: - description: Deployment selection in: body @@ -6714,7 +6714,7 @@ paths: - Sandbox Manager /core/v1/sandbox/runtime-observations: get: - description: Core key only. Each observation is labelled with its owning Project ID. Uses the existing read-only Runtime sampler, with bounded concurrency and no execution or provisioning. A provider with a batch metrics read, such as E2B, samples the page's running sandboxes in one bounded request. + description: Core key only. Each observation is labelled with its owning Project ID. Uses the existing read-only Runtime sampler, with bounded concurrency and no execution or provisioning. parameters: - description: Last Session ID from the preceding page in: query diff --git a/contracts/agents-api/execution-tools.md b/contracts/agents-api/execution-tools.md index 1b52e7a65..36ace1a5c 100644 --- a/contracts/agents-api/execution-tools.md +++ b/contracts/agents-api/execution-tools.md @@ -83,7 +83,7 @@ Execution admits only `mode: "disabled"` and `enabled: false`. Enabled or omitte } ``` -- `server_label` is nonempty and unique within the Session. Only the `http` transport is accepted; `server_url` is an absolute HTTP or HTTPS URL without credentials, query or fragment. Nonempty `headers` and `request_metadata` are rejected. +- `server_label` is nonempty and unique within the Session. Only the `http` transport is accepted; `server_url` is an absolute HTTP or HTTPS URL without credentials, query or fragment. Nonempty `headers` and `request_metadata` and an inline `authorization` are rejected. - [Public MCP connection origin](./environments.md#public-mcp-connection-origin) owns origin defaults, placement and credential authority; [Harness capabilities](./harness-capabilities.md#tools) owns per-Harness support. - Omitted or null `allowed_tools` permits every server tool; `[]` permits none. - `required: true` makes native thread creation and cold resume wait for the server to initialize; a failure stops execution without replacing retained history. It needs the Runtime's `mcp_http_required` capability. Public work can be accepted or queued during the wait. diff --git a/contracts/agents-api/go-bindings.json b/contracts/agents-api/go-bindings.json index 5b5d9253e..3e2178e08 100644 --- a/contracts/agents-api/go-bindings.json +++ b/contracts/agents-api/go-bindings.json @@ -159,10 +159,7 @@ "FunctionToolInput": { "sources": ["#/components/schemas/AgentToolConfigParamFunction"], "fields": { - "name": {"type": "*string"}, - "description": {"type": "*string"}, - "parameters": {"type": "json.RawMessage"}, - "defer_loading": {"type": "json.RawMessage"} + "parameters": {"type": "json.RawMessage"} } }, "InlineAgent": { @@ -233,16 +230,7 @@ "order": ["type", "server_label", "transport", "allowed_tools", "connection_origin", "credential_id", "request_metadata", "required"] }, "MCPToolInput": { - "sources": ["#/components/schemas/AgentToolConfigParamMcp"], - "fields": { - "server_label": {"type": "*string"}, - "allowed_tools": {"type": "json.RawMessage", "omit": false}, - "connection_origin": {"omit": false}, - "credential_id": {"omit": false}, - "request_metadata": {"type": "json.RawMessage", "omit": false}, - "required": {"type": "json.RawMessage", "omit": false} - }, - "order": ["type", "server_label", "transport", "allowed_tools", "connection_origin", "credential_id", "request_metadata", "required"] + "sources": ["#/components/schemas/AgentToolConfigParamMcp"] }, "MultiAgentConfig": { "sources": ["#/components/schemas/MultiAgentConfigResource"], diff --git a/contracts/agents-api/index.md b/contracts/agents-api/index.md index 88c1f7cfc..3f3158f0a 100644 --- a/contracts/agents-api/index.md +++ b/contracts/agents-api/index.md @@ -14,9 +14,9 @@ Core targets the complete OpenAI Agents API as pinned below ([public API rule](h | [openapi.yaml](./openapi.yaml) | The official public contract with Core's `x_agents_core` extension on Agent and Session request/response objects | | [go-bindings.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/go-bindings.json) | Go names, field representations, encoding order and stored projections; it does not define official field membership, enums or constraints | -Run `make openapi` to regenerate the public Go types, route inventory and all three OpenAPI documents. `scripts/generate-public-api.py` reads the checked-in, checksum-verified official source without network access. It selects Agents, Vaults, Files and Skills and follows their schema references, preserving union types, nullability, required fields and constraints. Core's extension types in `v1/` remain authored in Go and are added to the public schema during generation. The internal `/core/v1` and `/api/v1` documents come from handler annotations. `make check-openapi` checks freshness and the generator; it also runs through `make check-go`. +Run `make openapi` to regenerate the public Go types, the Agent request shapes, the route inventory and all three OpenAPI documents. `scripts/generate-public-api.py` reads the checked-in, checksum-verified official source without network access. It selects Agents, Vaults, Files and Skills and follows their schema references, preserving union types, nullability, required fields and constraints. The request shapes in `services/core/internal/api/official_shapes.gen.go` project `CreateAgentParams`, `UpdateAgentParams` and `SessionAgentConfigParam`; Core checks request bodies against them before it reads an Agent configuration. Core's extension types in `v1/` remain authored in Go and are added to the public schema during generation. The internal `/core/v1` and `/api/v1` documents come from handler annotations. `make check-openapi` checks freshness and the generator; it also runs through `make check-go`. -The public contract is the official API plus Core extensions. Standard fields are generated into `v1/official.gen.go`; `go-bindings.json` lists types consumed by Core and overrides only the Go representation or field order that existing storage or custom JSON encoding requires. Unspecified fields follow the official schema; shared shapes use one Go type. Selected discriminated unions also generate JSON serializers to retain required nullable fields for each variant. Other union serializers, request admission and state transitions remain implementation code. Contract tests verify that the public schema preserves the official definitions, extensions remain in `x_agents_core`, and all documents match registered routes. Official-client and raw HTTP tests verify behavior. Schema generation does not qualify an unimplemented feature; the gaps below still apply. Upstream upgrades update the OpenAPI and SDK pins together after comparison and compatibility tests. +The public contract is the official API plus Core extensions. Standard fields are generated into `v1/official.gen.go`; `go-bindings.json` lists types consumed by Core and overrides only the Go representation or field order that existing storage or custom JSON encoding requires. Unspecified fields follow the official schema; shared shapes use one Go type. Selected discriminated unions also generate JSON serializers to retain required nullable fields for each variant. Other union serializers, Core's local limits, execution admission and state transitions remain implementation code. Contract tests verify that the public schema preserves the official definitions, extensions remain in `x_agents_core`, and all documents match registered routes. Official-client and raw HTTP tests verify behavior. Schema generation does not qualify an unimplemented feature; the gaps below still apply. Upstream upgrades update the OpenAPI and SDK pins together after comparison and compatibility tests. The official source and existing service have these recorded differences: Agents authentication errors can return a null `code`; empty Files pages return null `first_id` and `last_id`; File resources can return null `expires_at` and `status_details`. The source declares those fields non-null. The official-client response validator allows null only for these named fields and otherwise validates OpenAPI 3.1 response schemas. Files and Skills operations omit error responses in the source, so those error bodies use the upstream shared `ErrorResponse` schema. [Wire semantics](./wire-semantics.md) and raw HTTP tests qualify service behavior; the published schema retains the official definitions. @@ -111,6 +111,7 @@ Each item is Core's deliberate or native behavior where the official service beh - Explicit reasoning effort or summary, service tiers other than `auto`, enabled `web_search` and enabled programmatic tool calling are saved but rejected at Session admission. - Harness support for tools, structured output, deferred discovery, subagents and MCP differs by Harness and placement; see the [Harness capabilities](./harness-capabilities.md). MiniMax Code has no public functions, no service-origin MCP and no image input. - Model-derived reasoning defaults are not resolved. +- MCP tools support the `http` transport only; `stdio` is rejected, and so is an inline `authorization` on a Session MCP transport ([HTTP MCP](./execution-tools.md#http-mcp)). **Execution and history** diff --git a/contracts/agents-api/runtime-observability-api.md b/contracts/agents-api/runtime-observability-api.md index 96ee57d21..2e1db7019 100644 --- a/contracts/agents-api/runtime-observability-api.md +++ b/contracts/agents-api/runtime-observability-api.md @@ -145,7 +145,7 @@ Only list rows carry `disk`: null, or `{usage_bytes, limit_bytes}` with the rule | `unsupported` | `runtime_mode_not_observable` | `none` and `self_hosted` Sessions. | | `unavailable` | `allocation_pending` | The managed allocation does not exist yet or is being created. | | `unavailable` | `runtime_not_running` | The allocation is being cleaned up or is released, or the provider reports the Runtime absent, stopped or suspended. | -| `unavailable` | `source_not_configured` | No observation source serves the allocation's provider. | +| `unavailable` | `source_not_configured` | This Core has no managed installation identity. | | `unavailable` | `sample_timeout` | The provider read exceeded its deadline. | | `unavailable` | `sample_unavailable` | The provider could not produce a current sample. | diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index 31fe8e4ff..a30353900 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -20,13 +20,11 @@ The resolver (`services/core/internal/deployment/observation.go`) reads the Sess Managed Docker, microsandbox and E2B allocations are observed. `none` and `self_hosted` Sessions are `unsupported`; Core never attributes shared host statistics to an `environment:none` Session. -The allocation's persisted `provider_key` selects exactly one configured source, which verifies the allocation's labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, `cleanup_pending` or `released` gives `runtime_not_running`, and a provider key without a source gives `source_not_configured`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read. +Every managed allocation is read through the deployment's selected Sandbox Provider, which verifies the allocation's installation (`provider_key`) and labels or equivalent ownership data before it returns values. Before any provider read, the allocation state decides some rows: `creating` or no allocation yet gives `allocation_pending`, `cleanup_pending` or `released` gives `runtime_not_running`, and a Core without an installation identity gives `source_not_configured`. A provider read that exceeds its deadline gives `sample_timeout`, a not-running result `runtime_not_running`, and an unavailable result `sample_unavailable`. Any other error, an ownership mismatch or an invalid sample fails the read. -The observation boundary is declared in `services/core/internal/runtimeobs/source.go`. Each registered `SourceResolver` declares supported `ResolveObservationSource`; registration validates this declaration without loading configuration or reading the database. Core resolves each provider key once per page, then validates the returned `Source` and uses that same immutable source for every read of the key on that page. An unconfigured resolver returns typed `ErrUnavailable`, which produces `sample_unavailable` without a provider type. Other resolution errors follow the provider-read error rules above. +`Observe` belongs to the [Sandbox Provider protocol](../../docs/sandbox-provider.md); `services/core/internal/runtimeobs/source.go` owns the observation types and the `Source` view of a Provider. Core loads the selected Provider and its registered kind once per page and uses that same immutable Provider for every read on that page, reading each running target with `Observe`. Without a selection the load returns typed `ErrUnavailable`, which produces `sample_unavailable` without a provider type. Other load errors follow the provider-read error rules above. -A source declares supported `ObservationProviderType`, which returns its immutable telemetry identity: a lowercase letter followed by at most 31 lowercase letters, digits or underscores. Empty identities are invalid. Provider registration and every resolved binding validate the identity and operation declarations before any sample or export. Reconfiguration affects later source resolutions; it cannot change the identity or provider selected for an in-flight page. Generation routers continue to resolve each allocation through its recorded deployment generation. - -A source implements `Observe` and declares `ObserveBatch` in its provider operations. When `ObserveBatch` is declared supported, one call reads up to 100 targets of that provider; when it is declared unsupported, Core reads each target with `Observe`. A failed batch read is never retried target by target. The [Sandbox Provider guide](../../docs/sandbox-provider.md) describes the operation declarations. +An observation's provider type is that registered kind: `docker`, `microsandbox` or `e2b`. Reconfiguration affects later pages; it cannot change the provider type or Provider selected for an in-flight page. Generation routers continue to resolve each allocation through its recorded deployment generation. ## Sample semantics @@ -56,7 +54,7 @@ Cumulative vCPU time, guest memory usage and the effective memory limit come fro ### E2B -One helper `observe` request reads a page of at most 100 allocations: E2B's batch metrics for the sandboxes named in the private receipts, and a labelled listing of the installation's running sandboxes that confirms each one. The [E2B helper](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/tools/e2b-provider/README.md) owns that request. It never connects to, renews or changes a sandbox and writes no receipts. +Core reads each allocation with one helper `observe` request: E2B's metrics for the sandbox named in its private receipt, and a listing of running sandboxes with the allocation's labels that confirms it. The [E2B helper](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/tools/e2b-provider/README.md) owns that request. It never connects to, renews or changes a sandbox and writes no receipts. | E2B value | Sample field | | --- | --- | @@ -65,11 +63,11 @@ One helper `observe` request reads a page of at most 100 allocations: E2B's batc | `memUsed`, `memTotal` | Memory usage and limit | | `diskUsed`, `diskTotal` | Disk usage and capacity, kept only when both are present and the total is nonzero | -E2B reports no cumulative CPU time, so CPU seconds stay null. `observed_at` is E2B's point time; a point up to 30 seconds ahead of Core's clock is recorded at Core's time, and a larger lead is `sample_unavailable`. A sandbox missing from the running listing is `runtime_not_running`. A missing or malformed point, an ambiguous listing and an E2B API failure, a rejected key included, are `sample_unavailable`; a malformed point affects only its own row. +E2B reports no cumulative CPU time, so CPU seconds stay null. `observed_at` is E2B's point time; a point up to 30 seconds ahead of Core's clock is recorded at Core's time, and a larger lead is `sample_unavailable`. A sandbox missing from the running listing is `runtime_not_running`. A missing or malformed point, an ambiguous listing and an E2B API failure, a rejected key included, are `sample_unavailable`. ## Read budgets -A current list read handles one page of up to 100 Sessions (default 20) with at most eight concurrent provider reads. Each provider read has two seconds, a batch read at least five, and the whole list request ten; beyond that the list returns 503. A single-Session read has two seconds. No provider call is retried within a request, and Core keeps no observation cache. +A current list read handles one page of up to 100 Sessions (default 20) with at most eight concurrent provider reads. Each provider read has two seconds and the whole list request ten; beyond that the list returns 503. A single-Session read has two seconds. No provider call is retried within a request, and Core keeps no observation cache. ## Durations @@ -123,7 +121,7 @@ With an OTLP endpoint configured, Core exports every record, both `on_read` and | `agents.session.tokens.input` | Gauge, tokens | Measured Session input tokens | | `agents.session.tokens.output` | Gauge, tokens | Measured Session output tokens | | `agents.runtime.sample` | Monotonic delta sum | One per validated result, unavailable and unsupported included | -| `agents.runtime.sample.duration` | Delta histogram, seconds | Provider read duration; a batch read counts once | +| `agents.runtime.sample.duration` | Delta histogram, seconds | Provider read duration | CPU and memory points are exported only when the sample has `started_at`; a missing measurement produces no point. Attributes are `agents.tenant.id`, `agents.session.id`, `agents.environment.id`, `agents.runtime.allocation.id`, `agents.runtime.mode`, `agents.runtime.provider.type`, `agents.runtime.status`, `agents.runtime.reason`, `agents.runtime.collection.source` and nanosecond `agents.runtime.resolved_at_unix_nano`, `agents.runtime.observed_at_unix_nano` and `agents.runtime.compute.started_at_unix_nano`. The nanosecond times keep records joinable when a backend stores event time at lower precision. Provider keys, receipts, native identifiers, raw errors, paths and credentials are never attributes. diff --git a/contracts/agents-api/v1/official.gen.go b/contracts/agents-api/v1/official.gen.go index 1cc7b4653..d74771d9c 100644 --- a/contracts/agents-api/v1/official.gen.go +++ b/contracts/agents-api/v1/official.gen.go @@ -273,10 +273,10 @@ type FunctionCallAction struct { // FunctionToolInput projects AgentToolConfigParamFunction. type FunctionToolInput struct { Type string `json:"type" binding:"required" enums:"function"` - Name *string `json:"name" binding:"required"` - Description *string `json:"description" binding:"required"` + Name string `json:"name" binding:"required"` + Description string `json:"description" binding:"required"` Parameters json.RawMessage `json:"parameters" binding:"required" swaggertype:"object"` - DeferLoading json.RawMessage `json:"defer_loading,omitempty" swaggertype:"object"` + DeferLoading *bool `json:"defer_loading,omitempty"` } // InlineAgent projects SessionAgentConfigParam. @@ -377,14 +377,14 @@ type MCPTool struct { // MCPToolInput projects AgentToolConfigParamMcp. type MCPToolInput struct { - Type string `json:"type" binding:"required" enums:"mcp"` - ServerLabel *string `json:"server_label" binding:"required"` - Transport json.RawMessage `json:"transport" binding:"required" swaggertype:"object"` - AllowedTools json.RawMessage `json:"allowed_tools" extensions:"x-nullable" swaggertype:"object"` - ConnectionOrigin *string `json:"connection_origin" extensions:"x-nullable" enums:"service,environment"` - CredentialID *string `json:"credential_id" extensions:"x-nullable"` - RequestMetadata json.RawMessage `json:"request_metadata" extensions:"x-nullable" swaggertype:"object"` - Required json.RawMessage `json:"required" swaggertype:"object"` + Type string `json:"type" binding:"required" enums:"mcp"` + ServerLabel string `json:"server_label" binding:"required"` + CredentialID *string `json:"credential_id,omitempty" extensions:"x-nullable"` + Transport json.RawMessage `json:"transport" binding:"required" swaggertype:"object"` + RequestMetadata map[string]json.RawMessage `json:"request_metadata,omitempty" extensions:"x-nullable" swaggertype:"object"` + AllowedTools []string `json:"allowed_tools,omitempty" extensions:"x-nullable"` + Required *bool `json:"required,omitempty"` + ConnectionOrigin *string `json:"connection_origin,omitempty" extensions:"x-nullable" enums:"service,environment"` } // MultiAgentConfig projects MultiAgentConfigResource. diff --git a/contracts/agents-api/wire-semantics.md b/contracts/agents-api/wire-semantics.md index 75d9136f2..2a62ed66b 100644 --- a/contracts/agents-api/wire-semantics.md +++ b/contracts/agents-api/wire-semantics.md @@ -162,7 +162,7 @@ Saving a value does not make it executable. Session creation admits a smaller se ### Configuration validation -Agent create and update bodies and the inline `agent` of Session create are checked against the pinned shapes of `tools`, `text`, `reasoning`, `service_tier`, `multi_agent`, `model`, `name` and `instructions`, before their parsers and before Harness admission. Failures return 400 with type and code `invalid_request_error`: +Agent create and update bodies and the inline `agent` of Session create are checked against the pinned shapes of `tools`, `text`, `reasoning`, `service_tier`, `multi_agent`, `model`, `name`, `instructions` and `metadata`, before their parsers and before Harness admission. Failures return 400 with type and code `invalid_request_error`: | Case | Param | Message | | --- | --- | --- | @@ -175,7 +175,7 @@ Agent create and update bodies and the inline `agent` of Session create are chec | Function `parameters` with a string root `type` other than `object` | null | `Invalid schema for function '': schema must be a JSON Schema of 'type: "object"', got 'type: ""'.` | | `text.format` JSON schema with a string root `type` other than `object` | null | `agent.text.format.schema must have top-level type "object"; got ""`, also on Agent requests | -Within one object Core reports a union's `type` first, then unknown members, then member values in document order, then missing members; tools before `text`, and the whole object before the duplicate and schema-root checks. Schemas without a string root `type` are not checked. Function and output schemas, MCP `transport`, `request_metadata`, `metadata` and `x_agents_core` keep their own parsers. Update bodies and the inline Session agent are validated before the Agent lookup, so owned, foreign, missing and malformed Agent IDs give the same response. +Within one object Core reports a union's `type` first, then unknown members, then member values in document order, then missing members in the pinned schema's order; tools before `text`, and the whole object before the duplicate and schema-root checks. Schemas without a string root `type` are not checked. Function and output schemas, `request_metadata` and `x_agents_core` keep their own parsers. Update bodies and the inline Session agent are validated before the Agent lookup, so owned, foreign, missing and malformed Agent IDs give the same response. Core saves values the pinned shapes allow even when it cannot run them: function names of any length, enabled programmatic tool calling, reasoning effort `max` and service tier `flex`. diff --git a/contracts/agents-api/zh/execution-tools.md b/contracts/agents-api/zh/execution-tools.md index aaab6a7c9..022a42ed5 100644 --- a/contracts/agents-api/zh/execution-tools.md +++ b/contracts/agents-api/zh/execution-tools.md @@ -1,7 +1,7 @@ --- title: "执行工具" source: contracts/agents-api/execution-tools.md -source_hash: 5e0b9ab1938ccd0ee7b22b1482a365bd54285c8c09fc4cb1ca172e34ab122ce2 +source_hash: eacb0d566f9d787f393c1d462351023a1ca6fe97ffb32773c8f23139833f5762 --- Agent 在 `tools` 中声明应用函数、控制项和 MCP 服务器，并可在 `text.format` 中声明输出 schema。本契约说明 Core 如何验证声明、哪些内容跨越 Runtime 边界，以及调用方如何恢复待执行操作。[Harness 能力](harness-capabilities.md)列出各 Harness 在不同部署位置支持的操作。原生工作区工具和 Environment Plugin MCP 属于 [Environment](environments.md#skills-plugins-and-environment-mcp)。 @@ -85,7 +85,7 @@ Core 发送 `PromptRequestPayload.ToolSearch` 和每个 `FunctionTool.DeferLoadi } ``` -- `server_label` 非空且在 Session 中唯一。仅接受 `http` 传输；`server_url` 为不带凭据、查询或片段的绝对 HTTP 或 HTTPS URL。非空 `headers` 和 `request_metadata` 被拒绝。 +- `server_label` 非空且在 Session 中唯一。仅接受 `http` 传输；`server_url` 为不带凭据、查询或片段的绝对 HTTP 或 HTTPS URL。非空 `headers`、`request_metadata` 和内联 `authorization` 被拒绝。 - [公开 MCP 连接来源](environments.md#public-mcp-connection-origin)定义来源默认值、部署位置和凭据权限；[Harness 能力](harness-capabilities.md#tools)定义各 Harness 支持范围。 - 省略或 null 的 `allowed_tools` 允许所有服务器工具；`[]` 不允许任何工具。 - `required: true` 使原生线程创建和冷恢复等待服务器初始化；失败会停止执行，不替换保留历史。它要求 Runtime 的 `mcp_http_required` 能力。等待期间公开工作可被接受或排队。 diff --git a/contracts/agents-api/zh/index.md b/contracts/agents-api/zh/index.md index c93f781f1..85e24eac5 100644 --- a/contracts/agents-api/zh/index.md +++ b/contracts/agents-api/zh/index.md @@ -1,7 +1,7 @@ --- title: "Agents API 覆盖台账" source: contracts/agents-api/index.md -source_hash: abca1a568c1d5b58a35e4c3b17a488ca8f9805fa114b13e6692c25699a77a7f8 +source_hash: fe62502746c71dd9444acab51d5bf9e6929107b5e905d0348078376c635faacc --- Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API（[public API rule](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#public-api)）。本台账记录 Core 对各项资源实现了哪些内容、哪些契约保存其详细信息，并列出相对于 OpenAI 服务的所有已知差异和所有未解决缺口。[API namespaces and credentials](../../../docs/zh/api/index.md) 说明谁调用哪些 API；[Agents API guide](../../../docs/zh/api/public-agent-api.md) 介绍使用方法。 @@ -16,9 +16,9 @@ Core 旨在以下方固定版本为准支持完整的 OpenAI Agents API（[publi | [openapi.yaml](../openapi.yaml) | 官方公共契约，并在 Agent 和 Session 请求及响应对象上加入 Core 的 `x_agents_core` 扩展 | | [go-bindings.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/go-bindings.json) | Go 名称、字段表示、编码顺序和存储投影；不定义官方字段集合、枚举或约束 | -运行 `make openapi` 重新生成公共 Go 类型、路由清单和三个 OpenAPI 文档。`scripts/generate-public-api.py` 读取仓库内经过校验和验证的官方源文件，无需网络。它选择 Agents、Vaults、Files 和 Skills，并跟随 schema 引用，保留联合类型、可空性、必填字段和约束。Core 在 `v1/` 中的扩展类型继续由 Go 定义，在生成时加入公共 schema。内部 `/core/v1` 和 `/api/v1` 文档由处理函数注解生成。`make check-openapi` 检查生成结果是否最新并测试生成器；`make check-go` 也会运行此检查。 +运行 `make openapi` 重新生成公共 Go 类型、Agent 请求结构、路由清单和三个 OpenAPI 文档。`scripts/generate-public-api.py` 读取仓库内经过校验和验证的官方源文件，无需网络。它选择 Agents、Vaults、Files 和 Skills，并跟随 schema 引用，保留联合类型、可空性、必填字段和约束。`services/core/internal/api/official_shapes.gen.go` 中的请求结构投影 `CreateAgentParams`、`UpdateAgentParams` 和 `SessionAgentConfigParam`；Core 在读取 Agent 配置前先按它们检查请求体。Core 在 `v1/` 中的扩展类型继续由 Go 定义，在生成时加入公共 schema。内部 `/core/v1` 和 `/api/v1` 文档由处理函数注解生成。`make check-openapi` 检查生成结果是否最新并测试生成器；`make check-go` 也会运行此检查。 -公共契约是官方 API 加上 Core 扩展。标准字段生成到 `v1/official.gen.go`；`go-bindings.json` 只列出 Core 使用的类型，仅在已有存储或自定义 JSON 编码需要时覆盖 Go 表示或字段顺序。未覆盖的字段遵循官方 schema，相同结构复用同一个 Go 类型。部分带判别字段的联合类型也从 schema 生成 JSON 序列化代码，保留每个分支必需的可空字段。其他联合类型序列化、请求准入和状态转换仍由实现代码负责。契约测试验证公共 schema 保留官方定义、扩展位于 `x_agents_core` 中，且所有文档与注册路由一致。官方客户端和原始 HTTP 测试验证行为。生成 schema 不代表某个尚未实现的功能已经得到验证；下方缺口仍然适用。升级上游时，在比对和兼容性测试后一起更新 OpenAPI 和 SDK 固定版本。 +公共契约是官方 API 加上 Core 扩展。标准字段生成到 `v1/official.gen.go`；`go-bindings.json` 只列出 Core 使用的类型，仅在已有存储或自定义 JSON 编码需要时覆盖 Go 表示或字段顺序。未覆盖的字段遵循官方 schema，相同结构复用同一个 Go 类型。部分带判别字段的联合类型也从 schema 生成 JSON 序列化代码，保留每个分支必需的可空字段。其他联合类型序列化、Core 的本地限制、执行准入和状态转换仍由实现代码负责。契约测试验证公共 schema 保留官方定义、扩展位于 `x_agents_core` 中，且所有文档与注册路由一致。官方客户端和原始 HTTP 测试验证行为。生成 schema 不代表某个尚未实现的功能已经得到验证；下方缺口仍然适用。升级上游时，在比对和兼容性测试后一起更新 OpenAPI 和 SDK 固定版本。 官方源文件与已有服务存在以下已记录的差异：Agents 鉴权错误的 `code` 可以为 null；Files 空页的 `first_id` 和 `last_id` 为 null；File 资源的 `expires_at` 和 `status_details` 可以为 null。源文件将这些字段声明为非空。官方客户端响应验证器只对这些指定字段允许 null，其余部分按 OpenAPI 3.1 响应 schema 验证。源文件中的 Files 和 Skills 操作未声明错误响应，因此这些错误体使用上游共享的 `ErrorResponse` schema。[传输语义](wire-semantics.md)和原始 HTTP 测试验证服务行为；发布的 schema 保留官方定义。 @@ -114,6 +114,7 @@ Core 自身字段位于 `x_agents_core` 中（[Core extensions](../../../docs/zh - 显式指定推理强度或摘要、使用 `auto` 之外的服务层级、启用 `web_search` 或启用程序化工具调用，这些设置都会被保存，但在 Session 准入时会被拒绝。 - Harness 对工具、结构化输出、延迟发现、subagents 和 MCP 的支持因 Harness 和部署位置而异；请参阅 [Harness capabilities](harness-capabilities.md)。MiniMax Code 不提供公共 functions、没有服务源 MCP，也不支持图像输入。 - 由模型推导出的推理默认值不会被解析确定。 +- MCP 工具仅支持 `http` 传输，`stdio` 会被拒绝，Session MCP 传输中的内联 `authorization` 也会被拒绝（[HTTP MCP](execution-tools.md#http-mcp)）。 **执行和历史** diff --git a/contracts/agents-api/zh/runtime-observability-api.md b/contracts/agents-api/zh/runtime-observability-api.md index d81203db3..984c5ab7e 100644 --- a/contracts/agents-api/zh/runtime-observability-api.md +++ b/contracts/agents-api/zh/runtime-observability-api.md @@ -1,7 +1,7 @@ --- title: "Runtime 遥测 API" source: contracts/agents-api/runtime-observability-api.md -source_hash: d6111447def530b07c392d457cce0cd553f92dafb2e7cc7f481da47f816fcab2 +source_hash: 6eca80ffefcaf8e26901659e5251518f84d2c6c93349085b378d46f9ac49149d --- Core 通过 `/core/v1` 下的只读管理员路由报告托管 Runtime 和沙箱节点所使用的信息：当前 Runtime 观测值、单个 Session 的已存储 Runtime 历史记录，以及沙箱节点的主机观测值和历史记录。读取操作绝不创建、唤醒、续期或更改计算资源，也绝不向历史记录添加样本。[Runtime observability](runtime-observability.md) 定义了 Core 如何采集和保留这些值；[Console API usage](../../../docs/zh/web/console-api-usage.md) 列出了读取这些值的 Web 页面。 @@ -147,7 +147,7 @@ Authorization: Bearer | `unsupported` | `runtime_mode_not_observable` | `none` 和 `self_hosted` Session。 | | `unavailable` | `allocation_pending` | 托管分配尚不存在或正在创建。 | | `unavailable` | `runtime_not_running` | 分配正在清理或已释放，或者提供方报告 Runtime 不存在、已停止或已暂停。 | -| `unavailable` | `source_not_configured` | 该分配的提供方未配置任何观测源。 | +| `unavailable` | `source_not_configured` | 此 Core 没有托管 installation 标识。 | | `unavailable` | `sample_timeout` | 提供方读取超过其截止时间。 | | `unavailable` | `sample_unavailable` | 提供方无法生成当前样本。 | diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index ddd10ea91..b4ef0f415 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -1,7 +1,7 @@ --- title: "运行时可观测性" source: contracts/agents-api/runtime-observability.md -source_hash: 103575f3971e77d5ba149cacb27e972e429a46713b2bda7da36cae43bbf313fa +source_hash: d18a476b06248dedfa5630ccf0f8a29dff59677a1e161d1847cc7d05e0c48de8 --- 这是面向贡献者的契约，规定 Core 如何观测 Runtime 并保留其历史。路由和响应字段见 [Runtime telemetry API](runtime-observability-api.md)。代码位于 `services/core/internal/runtimeobs`（解析、源、采样器和导出）、`internal/runtimehistory`（历史查询和 PostgreSQL 存储）以及 `internal/runtimeobs/otlpexporter`。 @@ -22,13 +22,11 @@ none: tenant_id -> session_id (no Session-owned Runtime instance) 托管 Docker、microsandbox 和 E2B 分配均会被观测。`none` 和 `self_hosted` Session 为 `unsupported`；Core 绝不会将共享主机统计信息归属于 `environment:none` Session。 -分配中持久化的 `provider_key` 会选择且仅选择一个已配置源；该源在返回数值前会验证分配标签或等效所有权数据。在读取任何 provider 之前，部分行的结果由分配状态决定：处于 `creating` 状态或尚无分配时得到 `allocation_pending`，处于 `cleanup_pending` 或 `released` 状态时得到 `runtime_not_running`，provider key 没有对应源时得到 `source_not_configured`。provider 读取超出截止时间时得到 `sample_timeout`，返回未运行结果时得到 `runtime_not_running`，返回不可用结果时得到 `sample_unavailable`。任何其他错误、所有权不匹配或无效采样都会使读取失败。 +每个托管分配都通过部署所选的 Sandbox Provider 读取；该 Provider 在返回数值前会验证分配的 installation（`provider_key`）以及分配标签或等效所有权数据。在读取任何 provider 之前，部分行的结果由分配状态决定：处于 `creating` 状态或尚无分配时得到 `allocation_pending`，处于 `cleanup_pending` 或 `released` 状态时得到 `runtime_not_running`，Core 没有 installation 标识时得到 `source_not_configured`。provider 读取超出截止时间时得到 `sample_timeout`，返回未运行结果时得到 `runtime_not_running`，返回不可用结果时得到 `sample_unavailable`。任何其他错误、所有权不匹配或无效采样都会使读取失败。 -观测边界在 `services/core/internal/runtimeobs/source.go` 中声明。每个注册的 `SourceResolver` 都声明支持 `ResolveObservationSource`；注册过程会验证此声明，但不会加载配置或读取数据库。Core 每页只解析每个 provider key 一次，随后验证返回的 `Source`，并在该页上针对此 key 的每次读取中使用同一不可变源。未配置的解析器返回类型化的 `ErrUnavailable`，从而生成不含 provider 类型的 `sample_unavailable`。其他解析错误遵循上述 provider 读取错误规则。 +`Observe` 属于 [Sandbox Provider 协议](../../../docs/zh/sandbox-provider.md)；`services/core/internal/runtimeobs/source.go` 负责观测类型以及 Provider 的 `Source` 视图。Core 每页只加载一次所选 Provider 及其注册 kind，并在该页的每次读取中使用同一不可变 Provider，用 `Observe` 读取每个运行中的目标。没有选择时，加载返回类型化的 `ErrUnavailable`，从而生成不含 provider 类型的 `sample_unavailable`。其他加载错误遵循上述 provider 读取错误规则。 -源会声明支持的 `ObservationProviderType`，该类型返回其不可变遥测标识：一个小写字母，后跟最多 31 个小写字母、数字或下划线。空标识无效。在任何采样或导出之前，provider 注册和每个解析后的绑定都会验证标识及操作声明。重新配置会影响后续的源解析，但无法更改正在处理页面所选定的标识或 provider。Generation 路由器仍会通过每个分配记录的部署代次解析该分配。 - -源实现 `Observe`，并在 provider 操作中声明 `ObserveBatch`。声明支持 `ObserveBatch` 时，一次调用可读取该 provider 的最多 100 个目标；声明不支持时，Core 使用 `Observe` 读取每个目标。批量读取失败后绝不会逐个重试目标。[Sandbox Provider guide](../../../docs/zh/sandbox-provider.md) 说明了这些操作声明。 +观测的 provider 类型即该注册 kind：`docker`、`microsandbox` 或 `e2b`。重新配置会影响后续页面，但无法更改正在处理页面所选定的 provider 类型或 Provider。Generation 路由器仍会通过每个分配记录的部署代次解析该分配。 ## 采样语义 {#sample-semantics} @@ -58,7 +56,7 @@ none: tenant_id -> session_id (no Session-owned Runtime instance) ### E2B {#e2b} -一次 helper `observe` 请求会读取一页最多 100 个分配：读取私有回执中指定 sandbox 的 E2B 批量指标，并获取该 installation 中运行中 sandbox 的带标签列表，以确认每一个 sandbox。[E2B helper](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/tools/e2b-provider/README.md) 负责此请求。它绝不会连接、续期或更改 sandbox，也不会写入任何回执。 +Core 用一次 helper `observe` 请求读取一个分配：读取私有回执中指定 sandbox 的 E2B 指标，并按该分配的标签列出运行中的 sandbox，以确认该 sandbox。[E2B helper](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/tools/e2b-provider/README.md) 负责此请求。它绝不会连接、续期或更改 sandbox，也不会写入任何回执。 | E2B 值 | 采样字段 | | --- | --- | @@ -67,11 +65,11 @@ none: tenant_id -> session_id (no Session-owned Runtime instance) | `memUsed`、`memTotal` | 内存使用量和限制 | | `diskUsed`、`diskTotal` | 磁盘使用量和容量；仅当两者都存在且总量非零时保留 | -E2B 不报告累计 CPU 时间，因此 CPU 秒数保持为 null。`observed_at` 为 E2B 的时间点；比 Core 时钟最多领先 30 秒的时间点按 Core 时间记录，领先幅度更大时则为 `sample_unavailable`。未出现在运行列表中的 sandbox 为 `runtime_not_running`。时间点缺失或格式错误、列表存在歧义以及 E2B API 失败（包括 key 被拒绝）均为 `sample_unavailable`；格式错误的时间点只影响其所在行。 +E2B 不报告累计 CPU 时间，因此 CPU 秒数保持为 null。`observed_at` 为 E2B 的时间点；比 Core 时钟最多领先 30 秒的时间点按 Core 时间记录，领先幅度更大时则为 `sample_unavailable`。未出现在运行列表中的 sandbox 为 `runtime_not_running`。时间点缺失或格式错误、列表存在歧义以及 E2B API 失败（包括 key 被拒绝）均为 `sample_unavailable`。 ## 读取预算 {#read-budgets} -当前列表读取处理一页最多 100 个 Session（默认 20 个），provider 读取并发数最多为 8。每次 provider 读取的时限为 2 秒，批量读取至少为 5 秒，整个列表请求为 10 秒；超过这些时限时，列表返回 503。单 Session 读取的时限为 2 秒。一次请求内不会重试任何 provider 调用，Core 也不保留观测缓存。 +当前列表读取处理一页最多 100 个 Session（默认 20 个），provider 读取并发数最多为 8。每次 provider 读取的时限为 2 秒，整个列表请求为 10 秒；超过这些时限时，列表返回 503。单 Session 读取的时限为 2 秒。一次请求内不会重试任何 provider 调用，Core 也不保留观测缓存。 ## 时长 {#durations} @@ -125,7 +123,7 @@ PostgreSQL 存储仅保留周期性的 `openai_hosted` 记录，因此 API 读 | `agents.session.tokens.input` | Gauge，令牌 | 实测 Session 输入令牌 | | `agents.session.tokens.output` | Gauge，令牌 | 实测 Session 输出令牌 | | `agents.runtime.sample` | 单调差值和 | 每个经验证的结果一条，包括 unavailable 和 unsupported | -| `agents.runtime.sample.duration` | Delta 直方图，秒 | Provider 读取时长；批量读取仅计一次 | +| `agents.runtime.sample.duration` | Delta 直方图，秒 | Provider 读取时长 | 仅当采样包含 `started_at` 时才导出 CPU 和内存数据点；缺少测量值不会产生数据点。属性包括 `agents.tenant.id`、`agents.session.id`、`agents.environment.id`、`agents.runtime.allocation.id`、`agents.runtime.mode`、`agents.runtime.provider.type`、`agents.runtime.status`、`agents.runtime.reason`、`agents.runtime.collection.source`，以及以纳秒为单位的 `agents.runtime.resolved_at_unix_nano`、`agents.runtime.observed_at_unix_nano` 和 `agents.runtime.compute.started_at_unix_nano`。这些纳秒时间使记录在后端以较低精度存储事件时间时仍可关联。Provider key、回执、原生标识符、原始错误、路径和凭据绝不会作为属性。 diff --git a/contracts/agents-api/zh/wire-semantics.md b/contracts/agents-api/zh/wire-semantics.md index 840c3a8f7..d688f9444 100644 --- a/contracts/agents-api/zh/wire-semantics.md +++ b/contracts/agents-api/zh/wire-semantics.md @@ -1,7 +1,7 @@ --- title: "Core 协议行为" source: contracts/agents-api/wire-semantics.md -source_hash: 5524db9b90aaf51026746316d6305da396033f50e88ed50792a8f52cf5aac9db +source_hash: bed64f05b6e52ab5774063f649173b4a42c0a2da4038bc273f595f81c5dbd128 --- 已锁定版本的 OpenAI Python SDK（[upstream.json](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/contracts/agents-api/upstream.json)）定义了 `/v1` 路由、字段和类型。本页面说明这些类型未作规定之处 Core 的行为，例如状态码、错误字段、默认值和列表边界，以及 Core 与官方服务存在差异的地方。[coverage ledger](index.md) 列出了这些差异和尚存缺口；[Sessions, events and history](sessions-events.md)、[message content](message-content.md)、[Vaults](vaults.md)、[source Files and Skills](source-files.md) 和 [Environment files and Artifacts](environment-files.md) 分别负责各自资源的规则。 @@ -164,7 +164,7 @@ Agent 创建要求提供 `model`。对于省略的字段，Core 会保存并返 ### 配置验证 {#configuration-validation} -Agent 创建和更新正文以及 Session 创建中的内联 `agent`，会在其解析器和 Harness 准入之前，根据已锁定的 `tools`、`text`、`reasoning`、`service_tier`、`multi_agent`、`model`、`name` 和 `instructions` 形状进行检查。失败时返回 400，`type` 和 `code` 均为 `invalid_request_error`： +Agent 创建和更新正文以及 Session 创建中的内联 `agent`，会在其解析器和 Harness 准入之前，根据已锁定的 `tools`、`text`、`reasoning`、`service_tier`、`multi_agent`、`model`、`name`、`instructions` 和 `metadata` 形状进行检查。失败时返回 400，`type` 和 `code` 均为 `invalid_request_error`： | 情况 | Param | 消息 | | --- | --- | --- | @@ -177,7 +177,7 @@ Agent 创建和更新正文以及 Session 创建中的内联 `agent`，会在其 | Function `parameters` 的字符串根 `type` 不是 `object` | null | `Invalid schema for function '': schema must be a JSON Schema of 'type: "object"', got 'type: ""'.` | | `text.format` JSON schema 的字符串根 `type` 不是 `object` | null | `agent.text.format.schema must have top-level type "object"; got ""`，Agent 请求上也会返回此消息 | -在同一个对象内，Core 会先报告联合类型的 `type`，然后报告未知成员，再按文档顺序报告成员值，最后报告缺失成员；先检查 tools，再检查 `text`，并在重复项和 schema 根检查之前检查整个对象。没有字符串根 `type` 的 schema 不会被检查。Function 和 output schema、MCP `transport`、`request_metadata`、`metadata` 和 `x_agents_core` 使用各自的解析器。更新正文和 Session 内联 Agent 会在查找 Agent 之前进行验证，因此属于当前租户、属于外部租户、缺失和格式错误的 Agent ID 会得到相同的响应。 +在同一个对象内，Core 会先报告联合类型的 `type`，然后报告未知成员，再按文档顺序报告成员值，最后按固定 schema 的顺序报告缺失成员；先检查 tools，再检查 `text`，并在重复项和 schema 根检查之前检查整个对象。没有字符串根 `type` 的 schema 不会被检查。Function 和 output schema、`request_metadata` 和 `x_agents_core` 使用各自的解析器。更新正文和 Session 内联 Agent 会在查找 Agent 之前进行验证，因此属于当前租户、属于外部租户、缺失和格式错误的 Agent ID 会得到相同的响应。 即使无法执行，Core 也会保存已锁定形状允许的值：任意长度的函数名称、已启用的 programmatic tool calling、reasoning effort `max` 和 service tier `flex`。 diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index 7b4774a29..08ea0f988 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -15,7 +15,7 @@ Core owns durable Environment, allocation, placement and cleanup state; the Prov ## Steps -1. **Read the contract.** Implement the five required operations and give an explicit decision for every extension interface in [Implement the interface](#implement-the-interface). +1. **Read the contract.** Implement every method, support the required operations and declare a decision for each of the others in [Implement the interface](#implement-the-interface). 2. **Write the adapter package** under `services/core/internal/sandbox/`: native SDK calls, ownership checks, identity translation and private configuration. Assert `var _ sandbox.SandboxProvider = (*YourAdapter)(nil)`. An out-of-process helper lives in `services/core/tools/-provider`. 3. **Register the kind** once, following [Register the provider kind](#register-the-provider-kind). Registration is explicit construction, not an init-time plugin registry. 4. **Label owned resources** with the provider ownership labels in the [Runtime names](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#openagentcore-runtime-names) table, and never accept older label names as a fallback. @@ -25,7 +25,7 @@ Core owns durable Environment, allocation, placement and cleanup state; the Prov ## Implement the interface -`SandboxProvider` has five required operations: +`sandbox_provider.go` holds the Core–Sandbox Provider protocol: the `SandboxProvider` interface for allocation, checkpoint and observation, its request and result types, and the setup-time `ConfigurationAdapter` with its typed errors. Value types that Core also uses beyond this boundary, such as `DeploymentSpec` and `CallFence`, live in their own files of the same package. Every method is required at compile time, and `ProviderOperations()` declares which ones the Provider supports. Five operations are always supported: | Operation | Purpose | | --- | --- | @@ -39,26 +39,23 @@ A backend without a native renewable lease, such as Docker, still keeps Core's h ### Explicit operation contracts -Every provider implements the methods of each interface below and returns a complete `ProviderOperations()` declaration. The interface methods are the operation inventory; `sandbox.ValidateOperations` checks the declaration against it without a second hand-kept list. +Every provider returns a complete `ProviderOperations()` declaration with one entry for each `SandboxProvider` method except `ProviderOperations` itself. The interface's method set is the operation inventory, and `sandbox.ValidateOperations` checks the declaration against it. The groups in the table below are lists in the protocol file, and a test keeps them equal to that method set, so a new method must join one of them. -| Contract | Requirement | Responsibility | +| Operations | Requirement | Responsibility | | --- | --- | --- | -| `sandbox.SandboxProvider` | All five operations supported | Allocation lifecycle and bounded commands | -| `sandbox.CheckpointProvider` | Explicit decision for every method, the same for all of them | Exact compute incarnations, capture and restore, retained-source resume and cleanup | -| `runtimeobs.Source` | Explicit decision | Ownership-checked read-only observations | -| `runtimeobs.BatchSource` | Explicit decision; requires `Source` | Bounded observations in input order, with per-target errors | -| `sandbox.SelectionDiscoverer` | Explicit decision | Read-only native configuration discovery before commit | -| `sandbox.CredentialVerifier` | Explicit decision | Verify access to owned resources without mutation | +| `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | +| `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | +| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | -`CheckpointProvider` adds `Initial` and `NewCompute` (construct compute references without allocating), `GetCompute`, `Suspend`, `Resume`, `ResumeCompute` (thaw only the same resident instance after an aborted pause), `KillCompute`, `DeleteSnapshot` and `RunCommandCompute`, which runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). +`Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). -Each declaration entry is `state: supported` with no reason, or `state: unsupported` with an authored reason code. Missing, zero, unknown or unsafe entries and missing methods fail validation. Adding a method to an interface requires an explicit decision and implementation in every adapter; never supply a base type or generate blanket unsupported implementations. +Each declaration entry is `state: supported` with no reason, or `state: unsupported` with an authored reason code. Missing, zero, unknown or unsafe entries fail validation. Adding a method to `SandboxProvider` requires an explicit decision and implementation in every adapter; never supply a base type or generate blanket unsupported implementations. An unsupported method returns `providercontract.UnsupportedError` before any native I/O. The error names the exact operation and a safe code, never a native message, resource identity, endpoint or credential. An empty result, a nil error, `Unavailable` or an unknown mutation outcome never stands in for unsupported, and the five required methods can never return it. -Each adapter owns one `Operations()` function, shared by its instance and its registration. `providers.ValidateBinding` checks both against the interfaces and each other, and Runtime admission and node generation loading also reject incomplete providers. An interface assertion establishes only method shape; callers use the declaration to decide support. +Each adapter owns one `Operations()` function, shared by its instance and its registration. `providers.ValidateBinding` checks both against the interface and each other, and Runtime admission and node generation loading also reject incomplete providers. Callers check the declaration with `providercontract.Require` before they call an operation, never a type assertion. -`ObserveBatch` returns an error, not an ambiguous boolean. Only a typed `UnsupportedError` for `ObserveBatch` permits per-target `Observe` calls; an unavailable service, timeout or other failure never does. Observation never renews, starts, prepares or stops compute; see the [observation contract](../contracts/agents-api/runtime-observability.md). +`Observe` reads one allocation and never renews, starts, prepares or stops compute; see the [observation contract](../contracts/agents-api/runtime-observability.md). Contract tests call every method declared unsupported with no native client configured, require its matching error and zero result, and reject incomplete or contradictory declarations. Supported behavior still needs native and lifecycle tests. @@ -110,9 +107,9 @@ Hosted and self-hosted Environments use the same Runtime preparation; a provider A new provider takes these steps: 1. Implement the operation contracts in the adapter package, with native contract tests. -2. Add its specification and resource validators and, for native resource discovery before commit, an optional read-only `SelectionDiscoverer`. Put native credential verification behind `CredentialVerifier`. -3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required and whether configuration discovery is supported. Also implement `ConfigurationDiscoverer`, even when discovery is unsupported: it validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for discovery and credential replacement. -4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Node proxy identity and checkpoint support read this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. +2. Add its specification and resource validators. +3. Implement `sandbox.ConfigurationAdapter` over a typed native configuration. `DecodeInput` strictly parses the separate public `configuration` and write-only `credential` objects of a request. `Encode` produces whitelisted public selectors, read-only observations and separate secret bytes, and never passes request JSON through. `Decode` restores stored selectors, and keeps access to owned resources, without remote admission or new template validation. `Normalize` copies its input before changing it. `ResolveChange`, `Equal` and `WithCredential` own inheritance, identity and credential composition. `Requirements` declares whether a credential and a public Core origin are required, and which setup operations are supported: `Discovery` for `DiscoverConfiguration`, `SelectionDiscovery` for `DiscoverSelection` and `CredentialVerification` for `VerifyCredential`. `DiscoverConfiguration` validates the query and returns a safe catalog, never a mutation or an admission decision, while Core keeps authorization, input limits and deadlines. `DiscoverSelection` resolves a candidate's omitted native values before commit, and `VerifyCredential` verifies a credential's access to owned resources without mutation. Both receive the candidate's `sandbox.DirectConfig` and build any native client for that call only. A node provider accepts only an empty public object, rejects credentials and returns Unsupported for every setup operation and for credential replacement. +4. Register its constructor, policies, configuration adapter, operation declaration and defaults in `providers/registry.go`. Its key is the provider kind, which also labels the Provider's observations, and checkpoint support reads this entry. The installer's projection combines the registered policies with the shared field bounds in `sandbox/deployment_contract.go`; regenerate it with `go run ./services/core/cmd/specification-contract -write`. 5. Supply the distribution artifacts for the adapter and its helper, and offer the provider to operators through the registered configuration contract. **Known design gap:** Web's setup views carry provider-specific options, such as E2B's views. Exposing another provider through that surface currently requires a shared Web edit. This coupling does not meet [Complexity stays in the adapter](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter); new integrations must express their configuration through the protocol and keep vendor-specific behavior in the adapter. Never add a Session or Turn scheduling path, a vendor column or API field, or a vendor switch in the store. @@ -126,17 +123,17 @@ A new provider takes these steps: - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The specification and resource validators, the configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial installer projection. - The Runtime input policy either accepts the pinned Runtime or gives the adapter's fixed reason for rejecting it, never both. -- Checkpoint support requires node mode and positive idle and retention defaults that fit Runtime durations; a provider without checkpoint support configures no suspension defaults. +- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. Checkpoint support also requires positive idle and retention defaults that fit Runtime durations, and a provider without checkpoint support configures no suspension defaults. -The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery` uses the shared supported or unsupported declaration with a safe reason. A new requirement field or discovery method needs an explicit validation update and never inherits an existing decision. Configuration discovery is distinct from resource selection discovery, and requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. +The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. ### Configuration storage and construction -Preview and persistence use `providers.Normalize` and `providers.Describe`. `SelectionDiscoverer` resolves omitted native values before commit, and the complete specification is validated again at persistence. `providers.ResolveChange` owns configuration inheritance, and comparisons use normalized selectors, so preview, retry and commit share the same defaults. The store owns transactions, credential encryption, generation fencing, resource ownership and generic object storage: only the adapter interprets `provider_config` and `provider_metadata`, and `provider_credential` holds ciphertext bound to the installation and generation. Retained generations keep their original public configuration and metadata and compose the current credential through the adapter, so a credential replacement never rewrites a retained selector. Database constraints check object structure, not the registration list. +Preview and persistence use `providers.Normalize` and `providers.Describe`. `providers.DiscoverSelection` resolves omitted native values before commit, and the complete specification is validated again at persistence. `providers.ResolveChange` owns configuration inheritance, and comparisons use normalized selectors, so preview, retry and commit share the same defaults. The store owns transactions, credential encryption, generation fencing, resource ownership and generic object storage: only the adapter interprets `provider_config` and `provider_metadata`, and `provider_credential` holds ciphertext bound to the installation and generation. Retained generations keep their original public configuration and metadata and compose the current credential through the adapter, so a credential replacement never rewrites a retained selector. Database constraints check object structure, not the registration list. A direct adapter with a credential verifies all retained generations and allocation references before a key is replaced. The common `sandbox.CallFence` excludes native calls and waits for helper completion, including calls whose callers timed out; execution invokes the prepared verification and fencing callbacks without branching on a vendor. -Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and the factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through `CheckpointProvider`, never through a provider name. +Vendor deployment validation and SDK setup stay at the construction boundary, and construction never creates an Environment. For node-local adapters `providers.Built` returns the provider, probe, installation identity, backend fingerprint and specification digest, and a `Quiescent` check when a helper can outlive its caller; generation collection waits for it. The factory also returns its close function. `execution.RuntimeProvider` binds the adapter to its kind, installation ID, backend fingerprint, generation, mode and node ownership; the database owns the selection, and the in-memory copy is never another authority. Docker and microsandbox run on nodes, and E2B is constructed directly. The node proxy exposes checkpoint operations only for a backend whose registered declaration supports them, and common lifecycle code admits suspension through the checkpoint declaration, never through a provider name. The backend fingerprint identifies a native resource namespace, not capacity. Core keeps deployment generations so that owned allocations keep resolving to their original backend; never repoint retained allocations at a replacement backend. @@ -144,7 +141,7 @@ The backend fingerprint identifies a native resource namespace, not capacity. Co Each node adapter's registration owns its typed `NodeArtifacts` declaration: logical distribution path, release filename suffix and installation role (`node`, `runtime`, `policy` or `image`). Registration rejects missing declarations, unsafe paths and unknown roles. `go run ./services/core/cmd/provider-artifacts -write` generates the shared Web catalog and Python projection. Run the command without `-write` to check freshness. Distribution packaging, Web availability and node installation read this projection; adding a provider's payload does not add a provider-name branch to those consumers. -The launcher supplies `sandbox.ProcessPaths` from the [derived process environment](./configuration.md). Core reads these paths once and passes them to direct construction and configuration discovery. They are fixed distribution properties, not deployment settings or user-selectable helper paths. Each adapter resolves its own relative helper and state locations; E2B uses `e2b/oac-e2b-provider` and `e2b/`. Missing or nonabsolute roots fail before helper execution. Provider construction and discovery never read process environment variables. +The launcher supplies `sandbox.ProcessPaths` from the [derived process environment](./configuration.md). Core reads these paths once and passes them to direct construction and setup operations. They are fixed distribution properties, not deployment settings or user-selectable helper paths. Each adapter resolves its own relative helper and state locations; E2B uses `e2b/oac-e2b-provider` and `e2b/`. Missing or nonabsolute roots fail before helper execution. Provider construction and discovery never read process environment variables. ## Managed lifecycle diff --git a/docs/web/console-api-usage.md b/docs/web/console-api-usage.md index b81088cf4..09df51a49 100644 --- a/docs/web/console-api-usage.md +++ b/docs/web/console-api-usage.md @@ -98,7 +98,7 @@ The list carries each harness's configuration, so the console does not read `GET | Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (`OAC_PUBLIC_URL`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows the shared client's fixed safe address-configuration message in the setup wizard, with Managed in System leading to System, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `metadata.template_build` (status, CPU, memory, disk) shows on System, the Sandbox configuration summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | | E2B discovery | `POST /core/v1/sandbox/providers/e2b/discovery` | The setup wizard lists the templates the entered E2B key can see, then the selected template's ready builds. The key travels only in these request bodies and the deployment write | | Reset | `POST`, `DELETE /core/v1/sandbox/deployment/reset` | Explicitly clear hosted resources, or cancel the remaining clear at the observed generation; show Core's remaining and offline projection | -| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health; an empty `core_url` (a node Core did not enroll) is unknown, not old. **Add node** follows only the node whose `enrollment_id` equals its command's | +| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health. **Add node** follows only the node whose `enrollment_id` equals its command's | | Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics node dialog: the host's CPU busy share and memory from its last heartbeat, and their history over the page's range. **Edit node** reads `host.effective_cpu_cores` and `host.total_memory_bytes` to show the host beside each sandbox's size and at most how many of those fit | | Allocations | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes page; Sandbox metrics. Under microsandbox, a node's page shows from `compute_phase_changed_at` how long each allocation has been in its compute phase and, while suspended, about when Core reclaims it (that time plus the deployment's `suspension.retention_seconds`); a null time shows a dash | | Enrollment | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**: the administrator sets the node's sandbox limits (`max_active`; `max_retained` only for microsandbox, equal to `max_active` for Docker) before Core issues a single-use token inside a command that verifies the installer checksum, with the command's `enrollment_id`, which the node it registers reports. The command runs the installer with sudo (a system service) and passes the token on standard input; root runs it directly. No ordinary-user installation or removal entry is exposed, and the log hint always names the system service. The command downloads the installer from the installation's `public_url`. No token is requested until the installation is read, when it cannot be read, when it is `local_only` (or its `public_url` is not an HTTPS origin), or when `/console/config` lists `node_artifacts` without the deployment's provider. The dialog reads both again on opening and when the window regains focus | diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 539343160..91dc2f563 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "添加 Sandbox Provider" source: docs/sandbox-provider.md -source_hash: 36ed532c778ec8c1c4c596a011a37613ed2aa0e6ffdf20854ea30ef9a8e0c953 +source_hash: 952ffc561734637ecbae9ad46782575feefaccdfe1c41ed8c6ae1a82c2ff170b --- **Sandbox Provider** 为 Core 管理的 Environment 提供 Runtime daemon 运行所需的外层计算资源，以及启动 daemon 的有界引导流程。本指南说明如何添加 Provider，并作为 Core 驱动 Provider 的参考。接口为 [`SandboxProvider`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/sandbox/sandbox_provider.go)。 @@ -17,7 +17,7 @@ Core 拥有持久 Environment、allocation、placement 和 cleanup 状态；Prov ## 步骤 {#steps} -1. **阅读契约。** 实现五项必需操作，并对[实现接口](#implement-the-interface)中的每个扩展接口作出明确决定。 +1. **阅读契约。** 实现每个方法，支持必需操作，并按[实现接口](#implement-the-interface)对其余每项操作声明决定。 2. **编写 adapter 包**，放在 `services/core/internal/sandbox/`：包括原生 SDK 调用、所有权检查、身份转换和私有配置。声明 `var _ sandbox.SandboxProvider = (*YourAdapter)(nil)`。进程外 helper 放在 `services/core/tools/-provider`。 3. **注册 kind** 一次，遵循[注册 provider kind](#register-the-provider-kind)。注册是明确构造，不是 init 时 plugin registry。 4. **标记所属资源**，使用 [Runtime 名称](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#openagentcore-runtime-names)表中的 provider ownership label，不接受旧 label 名称作为回退。 @@ -27,7 +27,7 @@ Core 拥有持久 Environment、allocation、placement 和 cleanup 状态；Prov ## 实现接口 {#implement-the-interface} -`SandboxProvider` 有五项必需操作： +`sandbox_provider.go` 包含 Core–Sandbox Provider 协议：负责 allocation、checkpoint 和观测的 `SandboxProvider` 接口及其请求与结果类型，以及 setup 阶段的 `ConfigurationAdapter` 及其类型化错误。Core 在此边界之外也使用的值类型，例如 `DeploymentSpec` 和 `CallFence`，位于同一 package 的独立文件中。每个方法在编译期都必须实现，`ProviderOperations()` 声明 Provider 支持哪些方法。以下五项操作始终支持： | 操作 | 用途 | | --- | --- | @@ -41,26 +41,23 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir ### 明确的操作契约 {#explicit-operation-contracts} -每个 provider 实现以下各接口的方法，并返回完整的 `ProviderOperations()` 声明。接口方法就是操作清单；`sandbox.ValidateOperations` 根据接口检查声明，无需第二份手动维护的清单。 +每个 provider 返回完整的 `ProviderOperations()` 声明，`SandboxProvider` 中除 `ProviderOperations` 本身外的每个方法各占一项。接口的方法集就是操作清单，`sandbox.ValidateOperations` 根据它检查声明。下表中的分组是协议文件中的列表，由测试保证它们与该方法集一致，因此新增方法必须加入其中一组。 -| 契约 | 要求 | 职责 | +| 操作 | 要求 | 职责 | | --- | --- | --- | -| `sandbox.SandboxProvider` | 支持全部五项操作 | Allocation 生命周期与有界命令 | -| `sandbox.CheckpointProvider` | 对每个方法明确决定，所有方法一致 | 精确计算实例、捕获与恢复、保留源恢复和清理 | -| `runtimeobs.Source` | 明确决定 | 检查所有权的只读观测 | -| `runtimeobs.BatchSource` | 明确决定；要求 `Source` | 按输入顺序提供有界观测，包含每目标错误 | -| `sandbox.SelectionDiscoverer` | 明确决定 | 提交前只读原生配置发现 | -| `sandbox.CredentialVerifier` | 明确决定 | 验证对所属资源的访问，不修改资源 | +| `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | +| `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | +| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致；仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | -`CheckpointProvider` 增加 `Initial` 和 `NewCompute`（构造 compute reference，不分配资源）、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`（暂停中止后仅解冻同一驻留实例）、`KillCompute`、`DeleteSnapshot` 和 `RunCommandCompute`，后者在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 +`Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 -每个声明项为不带 reason 的 `state: supported`，或带 authored reason code 的 `state: unsupported`。缺失、零值、未知或不安全项以及缺失方法都会验证失败。给接口添加方法时，必须在每个 adapter 中明确决定并实现；不提供 base type，也不生成笼统的不支持实现。 +每个声明项为不带 reason 的 `state: supported`，或带 authored reason code 的 `state: unsupported`。缺失、零值、未知或不安全项都会验证失败。给 `SandboxProvider` 添加方法时，必须在每个 adapter 中明确决定并实现；不提供 base type，也不生成笼统的不支持实现。 不支持的方法在任何原生 I/O 前返回 `providercontract.UnsupportedError`。错误指明精确操作和安全 code，不包含原生消息、资源身份、endpoint 或凭据。空结果、nil error、`Unavailable` 或未知 mutation 结果都不能代替 unsupported，五项必需方法不能返回 unsupported。 -每个 adapter 拥有一个 `Operations()` 函数，由实例和注册共享。`providers.ValidateBinding` 根据接口并相互对照检查两者，Runtime admission 与 node generation 加载也拒绝不完整 provider。interface assertion 仅证明方法形态；调用方使用声明决定支持情况。 +每个 adapter 拥有一个 `Operations()` 函数，由实例和注册共享。`providers.ValidateBinding` 根据接口并相互对照检查两者，Runtime admission 与 node generation 加载也拒绝不完整 provider。调用方在调用操作前用 `providercontract.Require` 检查声明，从不使用 type assertion。 -`ObserveBatch` 返回 error，不返回有歧义的 boolean。仅 `ObserveBatch` 的类型化 `UnsupportedError` 允许逐目标调用 `Observe`；服务不可用、超时或其他失败都不允许。观测不续期、启动、准备或停止计算资源；参见[观测契约](../../contracts/agents-api/zh/runtime-observability.md)。 +`Observe` 读取一个 allocation，不续期、启动、准备或停止计算资源；参见[观测契约](../../contracts/agents-api/zh/runtime-observability.md)。 契约测试在未配置原生 client 时调用每个声明不支持的方法，要求匹配错误和零值结果，并拒绝不完整或矛盾的声明。支持的行为仍需要原生和生命周期测试。 @@ -112,9 +109,9 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` 新 provider 执行以下步骤： 1. 在 adapter 包中实现 operation 契约，并编写原生契约测试。 -2. 添加 specification 和 resource validator；提交前需要原生资源发现时，添加可选只读 `SelectionDiscoverer`。原生凭据验证放在 `CredentialVerifier` 后。 -3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及是否支持配置发现。即使不支持 discovery，也实现 `ConfigurationDiscoverer`：验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。node provider 仅接受空公开对象，拒绝凭据，对 discovery 和 credential replacement 返回 Unsupported。 -4. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter、operation 声明和默认值。Node proxy identity 和 checkpoint 支持读取此项。installer 投影组合已注册 policy 与 `sandbox/deployment_contract.go` 中的共享 field bound；通过 `go run ./services/core/cmd/specification-contract -write` 重新生成。 +2. 添加 specification 和 resource validator。 +3. 基于类型化原生配置实现 `sandbox.ConfigurationAdapter`。`DecodeInput` 严格解析请求中独立的公开 `configuration` 与只写 `credential` 对象。`Encode` 生成白名单公开 selector、只读观测和独立 secret bytes，不透传请求 JSON。`Decode` 恢复已存储 selector 并保留对所属资源的访问，不做远程 admission 或新模板验证。`Normalize` 修改前复制输入。`ResolveChange`、`Equal` 和 `WithCredential` 负责继承、身份与凭据组合。`Requirements` 声明是否需要凭据和公开 Core origin，以及支持哪些 setup 操作：`Discovery` 对应 `DiscoverConfiguration`，`SelectionDiscovery` 对应 `DiscoverSelection`，`CredentialVerification` 对应 `VerifyCredential`。`DiscoverConfiguration` 验证 query 并返回安全 catalog，不做 mutation 或 admission decision；Core 保留授权、输入限制与 deadline。`DiscoverSelection` 在提交前解析候选项省略的原生值，`VerifyCredential` 验证凭据对所属资源的访问，不修改资源。两者都接收候选项的 `sandbox.DirectConfig`，原生 client 只为该次调用构造。node provider 仅接受空公开对象，拒绝凭据，对每项 setup 操作和 credential replacement 返回 Unsupported。 +4. 在 `providers/registry.go` 中注册 constructor、policy、configuration adapter、operation 声明和默认值。其键即 provider kind，也用于标记该 Provider 的观测；checkpoint 支持读取此项。installer 投影组合已注册 policy 与 `sandbox/deployment_contract.go` 中的共享 field bound；通过 `go run ./services/core/cmd/specification-contract -write` 重新生成。 5. 提供 adapter 和 helper 的发行产物，通过已注册 configuration 契约向运维人员提供 provider。 **已知设计缺口：** Web 的 setup view 携带 provider 专有选项，如 E2B 的 view。通过该界面提供另一 provider 目前需要修改共享的 Web。此耦合不符合[复杂性留在 adapter 内](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/AGENTS.md#complexity-stays-in-the-adapter)；新集成必须通过协议表达配置，把厂商专有行为留在 adapter。不得添加 Session 或 Turn 调度路径、厂商专有 column 或 API field，或 store 中的厂商 switch。 @@ -128,17 +125,17 @@ Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity` - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - specification 和 resource validator、configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分 installer projection。 - Runtime input policy 要么接受固定 Runtime，要么给出 adapter 拒绝它的固定原因，不能两者兼有。 -- Checkpoint 支持要求 node mode 和适合 Runtime duration 的正 idle、retention 默认值；不支持 checkpoint 的 provider 不配置 suspension 默认值。 +- Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` Provider 会被注册拒绝。Checkpoint 支持还要求适合 Runtime duration 的正 idle、retention 默认值，不支持 checkpoint 的 provider 不配置 suspension 默认值。 -configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 或 discovery method 需要明确更新验证，不继承已有决定。configuration discovery 与 resource selection discovery 不同，要求凭据也不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 +configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 ### 配置存储与构造 {#configuration-storage-and-construction} -预览和持久化使用 `providers.Normalize` 与 `providers.Describe`。`SelectionDiscoverer` 在提交前解析省略的原生值，持久化时再次验证完整 specification。`providers.ResolveChange` 负责配置继承，比较使用 normalized selector，使预览、重试和提交共享默认值。store 负责事务、凭据加密、generation fencing、资源所有权和通用对象存储：仅 adapter 解释 `provider_config` 与 `provider_metadata`，`provider_credential` 保存绑定到安装实例与 generation 的密文。保留 generation 保持原公开配置和 metadata，通过 adapter 组合当前凭据，因此替换凭据不重写保留 selector。数据库约束检查对象结构，不检查注册列表。 +预览和持久化使用 `providers.Normalize` 与 `providers.Describe`。`providers.DiscoverSelection` 在提交前解析省略的原生值，持久化时再次验证完整 specification。`providers.ResolveChange` 负责配置继承，比较使用 normalized selector，使预览、重试和提交共享默认值。store 负责事务、凭据加密、generation fencing、资源所有权和通用对象存储：仅 adapter 解释 `provider_config` 与 `provider_metadata`，`provider_credential` 保存绑定到安装实例与 generation 的密文。保留 generation 保持原公开配置和 metadata，通过 adapter 组合当前凭据，因此替换凭据不重写保留 selector。数据库约束检查对象结构，不检查注册列表。 具有凭据的 direct adapter 在替换 key 前验证全部保留 generation 与 allocation reference。公共 `sandbox.CallFence` 排除原生调用并等待 helper 完成，包括调用方已超时的调用；execution 调用已准备的 verification 和 fencing callback，不按厂商分支。 -厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `providers.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest，factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 checkpoint 的 backend 暴露 checkpoint 操作，公共 lifecycle 通过 `CheckpointProvider` 准入 suspension，不通过 provider name。 +厂商部署验证和 SDK setup 留在构造边界，构造不创建 Environment。node-local adapter 的 `providers.Built` 返回 provider、probe、installation identity、backend fingerprint 和 specification digest；helper 可能比调用方存活更久时，还返回 `Quiescent` 检查，generation 回收会等待它。factory 还返回 close 函数。`execution.RuntimeProvider` 将 adapter 绑定到 kind、installation ID、backend fingerprint、generation、mode 和 node ownership；选择由数据库负责，内存副本不构成另一权限来源。Docker 与 microsandbox 在 node 上运行，E2B 直接构造。node proxy 仅对注册声明支持 checkpoint 的 backend 暴露 checkpoint 操作，公共 lifecycle 通过 checkpoint 声明准入 suspension，不通过 provider name。 backend fingerprint 标识原生资源命名空间，不表示容量。Core 保留部署 generation，使所属 allocation 继续解析到原 backend；不要将保留 allocation 重新指向替代 backend。 @@ -146,7 +143,7 @@ backend fingerprint 标识原生资源命名空间，不表示容量。Core 保 每个 node adapter 注册负责其类型化 `NodeArtifacts` 声明：逻辑发行路径、release filename suffix 和安装角色（`node`、`runtime`、`policy` 或 `image`）。注册拒绝缺失声明、不安全路径和未知角色。`go run ./services/core/cmd/provider-artifacts -write` 生成共享 Web catalog 和 Python projection。不带 `-write` 运行可检查是否最新。发行打包、Web availability 和 node 安装读取此投影；添加 provider payload 不在这些消费者中增加 provider-name 分支。 -launcher 从[派生进程环境](configuration.md)提供 `sandbox.ProcessPaths`。Core 读取这些路径一次，并传给 direct construction 和 configuration discovery。它们是固定发行属性，不是部署设置或用户可选 helper 路径。每个 adapter 解析自己的相对 helper 和 state 位置；E2B 使用 `e2b/oac-e2b-provider` 和 `e2b/`。root 缺失或不是绝对路径时，在执行 helper 前失败。Provider 构造与发现不读取进程环境变量。 +launcher 从[派生进程环境](configuration.md)提供 `sandbox.ProcessPaths`。Core 读取这些路径一次，并传给 direct construction 和 setup 操作。它们是固定发行属性，不是部署设置或用户可选 helper 路径。每个 adapter 解析自己的相对 helper 和 state 位置；E2B 使用 `e2b/oac-e2b-provider` 和 `e2b/`。root 缺失或不是绝对路径时，在执行 helper 前失败。Provider 构造与发现不读取进程环境变量。 ## 托管生命周期 {#managed-lifecycle} diff --git a/docs/zh/web/console-api-usage.md b/docs/zh/web/console-api-usage.md index a6037092a..1b4890fd5 100644 --- a/docs/zh/web/console-api-usage.md +++ b/docs/zh/web/console-api-usage.md @@ -1,7 +1,7 @@ --- title: "控制台 API 使用" source: docs/web/console-api-usage.md -source_hash: 50edc63c79976e9aad9590604a0e361aae178144bc8a6009989b2da5d031408d +source_hash: 7318d1d082d19cc1681a2bd91556c4dc7e16b211ae7cd3936b99d63c26949900 --- 本页列出各控制台页面读取和写入的 Core 路由，以及控制台如何限定读取范围。[administrator API contract](../../../contracts/agents-api/zh/admin-api.md) 定义了路由、响应结构、分页和审计记录；[API namespaces and credentials](../api/index.md) 定义了本文使用的术语。 @@ -100,7 +100,7 @@ source_hash: 50edc63c79976e9aad9590604a0e361aae178144bc8a6009989b2da5d031408d | 部署 | `GET`、`POST`、`PUT /core/v1/sandbox/deployment` | 读取提供商、只读 `core_url`（即 `OAC_PUBLIC_URL`，会显示在设置审核中且绝不发送）、重置状态、安装 ID 和规范；409 `sandbox_configuration_error`（E2B 搭配回环地址形式的 `public_url`）会在设置向导中显示共享客户端固定的安全地址配置消息，并通过 Managed in System 前往 System，且无需确认；使用 `resources` 以及 Docker 或 microsandbox 的 `runtime` release 初始化部署，或者使用 E2B 账户且不提供 `resources`（Core 采用模板构建的 CPU 和内存）；使用预期的 generation 更改设置。E2B 的 `metadata.template_build`（状态、CPU、内存、磁盘）会显示在 System、Sandbox 配置摘要和 Sandbox metrics 中；当缺少 `specification.resources` 时，它还会确定每个 Sandbox 的大小；microsandbox 的 `suspension`（空闲和保留秒数）会显示在 System 和 Nodes 摘要中 | | E2B 发现 | `POST /core/v1/sandbox/providers/e2b/discovery` | 设置向导先列出输入的 E2B 密钥可见的模板，再列出所选模板的可用构建。该密钥只会通过这些请求体和部署写入请求传输 | | 重置 | `POST`、`DELETE /core/v1/sandbox/deployment/reset` | 显式清除托管资源，或在观测到的 generation 处取消剩余清除；显示 Core 的剩余资源和离线预测 | -| Nodes | `GET /core/v1/sandbox/nodes` | Nodes 页面；Overview 上的机群；Sandbox metrics 中的节点容量。在线节点的 `diagnostic`（`docker_unavailable`、`docker_limits_unsupported`、`runtime_image_unavailable`、`kvm_unavailable`、`microsandbox_artifacts_unavailable`、`capacity_insufficient`、`provider_unavailable`；任何其他值均读取为 `provider_unavailable`）会将其标记为降级，并在上述每个页面及节点页面中，紧邻状态的帮助提示里说明原因和修复方法。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，Nodes 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 Old address，而不是健康状态；如果 `core_url` 为空（Core 未注册该节点），则状态为未知，而不是旧地址。**Add node** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | +| Nodes | `GET /core/v1/sandbox/nodes` | Nodes 页面；Overview 上的机群；Sandbox metrics 中的节点容量。在线节点的 `diagnostic`（`docker_unavailable`、`docker_limits_unsupported`、`runtime_image_unavailable`、`kvm_unavailable`、`microsandbox_artifacts_unavailable`、`capacity_insufficient`、`provider_unavailable`；任何其他值均读取为 `provider_unavailable`）会将其标记为降级，并在上述每个页面及节点页面中，紧邻状态的帮助提示里说明原因和修复方法。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，Nodes 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 Old address，而不是健康状态。**Add node** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | | 节点详情 | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics 节点对话框：主机自最近一次心跳以来的 CPU 忙碌占比和内存使用量，以及页面所选范围内二者的历史记录。**Edit node** 读取 `host.effective_cpu_cores` 和 `host.total_memory_bytes`，用于在每个 Sandbox 大小旁显示主机容量，以及最多可容纳多少个该大小的 Sandbox | | 分配 | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes 页面；Sandbox metrics。在 microsandbox 下，节点页面根据 `compute_phase_changed_at` 显示每个分配处于计算阶段的时间，并在分配暂停时估算 Core 回收它的时间（该时间加上部署的 `suspension.retention_seconds`）；时间为 null 时显示短横线 | | 注册 | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**：管理员先设置节点的 Sandbox 限制（`max_active`；`max_retained` 仅适用于 microsandbox，在 Docker 下等于 `max_active`），然后 Core 才会把一次性令牌放入命令中；该命令会验证安装程序校验和，并包含命令的 `enrollment_id`，节点注册时会报告此 ID。命令使用 sudo 运行安装程序（作为系统服务），并通过标准输入传递令牌；以 root 运行时则直接执行。界面不提供普通用户安装或移除入口，日志提示始终指明系统服务。命令从安装的 `public_url` 下载安装程序。只有成功读取安装信息后才会请求令牌；如果安装信息无法读取、安装为 `local_only`（或其 `public_url` 不是 HTTPS 来源），或者 `/console/config` 列出的 `node_artifacts` 不包含部署的提供商，则不会请求令牌。对话框在打开时和窗口重新获得焦点时，会再次读取这两项信息 | diff --git a/scripts/generate-public-api.py b/scripts/generate-public-api.py index 69081f3e8..f469e0f9b 100644 --- a/scripts/generate-public-api.py +++ b/scripts/generate-public-api.py @@ -10,6 +10,9 @@ ROOT = Path(__file__).resolve().parents[1] CONTRACT = ROOT / 'contracts/agents-api' +SHAPES = ROOT / 'services/core/internal/api/official_shapes.gen.go' +# Agent configuration request bodies checked by Core's shape walker. +REQUEST_SHAPES = ('CreateAgentParams', 'UpdateAgentParams', 'SessionAgentConfigParam') HTTP_METHODS = {'get', 'post', 'put', 'patch', 'delete', 'head', 'options'} @@ -173,9 +176,71 @@ def go_types(document, bindings): lines.append('}\n') if binding.get('marshal_union'): lines.extend(union_marshaler(document, name, binding, field_types)) + return gofmt(lines) + + +def gofmt(lines): return subprocess.run(['gofmt'], input='\n'.join(lines).encode(), stdout=subprocess.PIPE, check=True).stdout +def go_shape(document, schema, required=False): + """Project one request value onto a shape of services/core/internal/api/configuration_validation.go.""" + value, = [v for v in dereference(document, schema).get('anyOf', [schema]) if v.get('type') != 'null'] + value = dereference(document, value) + kind = value.get('type') + if isinstance(kind, list): + kind, = set(kind) - {'null'} + if 'oneOf' in value: + if value['discriminator']['propertyName'] != 'type': + raise ValueError(f'Unsupported union discriminator: {value["discriminator"]}') + variants = {} + for option in value['oneOf']: + option = dereference(document, option) + tag, = option['properties']['type']['enum'] + variants[tag] = go_members(document, option, 'type') + fields = ['kind: unionValue', 'values: []string{' + ', '.join(json.dumps(tag) for tag in variants) + '}', + 'variants: map[string][]member{\n' + ''.join(f'{json.dumps(tag)}: {members.removeprefix("[]member")},\n' for tag, members in variants.items()) + '}'] + elif 'enum' in value: + fields = ['kind: enumValue', 'values: []string{' + ', '.join(json.dumps(v) for v in value['enum']) + '}'] + elif kind == 'object' and 'properties' in value: + fields = ['kind: objectValue', 'members: ' + go_members(document, value)] + elif kind == 'object': + values = value['additionalProperties'] + if values and values.get('type') != 'string': + raise ValueError(f'Unsupported map values: {values}') + fields = ['kind: mapValue'] + (['items: &shape{kind: stringValue}'] if values else []) + elif kind == 'array': + fields = ['kind: arrayValue', 'items: &' + go_shape(document, value['items'])] + elif kind == 'integer': + fields = ['kind: integerValue', f'minimum: {value["minimum"]}'] + else: + fields = [{'string': 'kind: stringValue', 'boolean': 'kind: booleanValue'}[kind]] + if required: + fields.append('required: true') + if nullable(document, schema): + fields.append('nullable: true') + return 'shape{' + ', '.join(fields) + '}' + + +def go_members(document, schema, skip=None): + # The walker rejects unknown members, so only closed objects project. + if schema.get('additionalProperties') is not False: + raise ValueError(f'Unsupported open object: {sorted(schema["properties"])}') + required = schema.get('required', []) + return '[]member{\n' + ''.join(f'{{{json.dumps(name)}, {go_shape(document, value, name in required)}}},\n' for name, value in schema['properties'].items() if name != skip) + '}' + + +def request_shapes(document, owners): + lines = ['// Code generated by scripts/generate-public-api.py; DO NOT EDIT.', 'package api', '', + '// Pinned request shapes; x_agents_core is checked by its own parser.', 'var ('] + for name in REQUEST_SHAPES: + members = go_members(document, resolve(document, '#/components/schemas/' + name)) + if name in owners: + members = members[:-1] + '{"x_agents_core", shape{}},\n}' + lines.append(f'{name[0].lower() + name[1:]} = shape{{kind: objectValue, members: {members}}}') + return gofmt(lines + [')']) + + def prune_components(document): needed = set() def walk(value): @@ -263,6 +328,7 @@ def main(): owners = extension_owners(bindings) if args.swag_roots: write(CONTRACT / 'v1/official.gen.go', go_types(source, bindings), args.check) + write(SHAPES, request_shapes(source, owners), args.check) roots = 'package extensions\n\n' + '\n'.join(f'// @Success 200 {{object}} v1.{name}' for name in sorted(set(owners.values()))) + '\nfunc extensions() {}\n' args.swag_roots.write_text(roots) else: diff --git a/scripts/generate-public-api.test.py b/scripts/generate-public-api.test.py index 531d205f8..ad47f433f 100644 --- a/scripts/generate-public-api.test.py +++ b/scripts/generate-public-api.test.py @@ -51,6 +51,25 @@ def test_new_official_fields_reach_go_without_a_second_field_list(self): self.assertIn('FutureField *string', generated) self.assertIn('json:"future_field,omitempty"', generated) + def test_generated_request_shapes_are_current(self): + owners = generator.extension_owners(self.bindings) + self.assertEqual(generator.request_shapes(self.source, owners), generator.SHAPES.read_bytes()) + + def test_new_official_members_and_values_reach_request_shapes(self): + source = copy.deepcopy(self.source) + schemas = source['components']['schemas'] + schemas['McpTransportConfigParamStdio']['properties']['future_field'] = {'type': 'string'} + schemas['ServiceTierParam']['enum'].append('future_tier') + generated = generator.request_shapes(source, generator.extension_owners(self.bindings)).decode() + self.assertIn('{"future_field", shape{kind: stringValue}}', generated) + self.assertEqual(generated.count('"future_tier"'), 3) + + def test_open_request_objects_are_rejected(self): + source = copy.deepcopy(self.source) + del source['components']['schemas']['McpTransportConfigParamStdio']['additionalProperties'] + with self.assertRaisesRegex(ValueError, 'Unsupported open object'): + generator.request_shapes(source, generator.extension_owners(self.bindings)) + def test_stale_binding_cannot_add_a_public_field(self): bindings = copy.deepcopy(self.bindings) bindings['Vault']['fields'] = {'private_data': {'type': 'string'}} diff --git a/services/core/cmd/sandbox-node/generations.go b/services/core/cmd/sandbox-node/generations.go index 3cd17d93f..78f41af37 100644 --- a/services/core/cmd/sandbox-node/generations.go +++ b/services/core/cmd/sandbox-node/generations.go @@ -171,7 +171,7 @@ func buildGeneration(registry *providerconfig.Registry, config providerconfig.Co if err != nil { return node.GenerationProvider{}, err } - return node.GenerationProvider{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, Provider: built.Provider, Probe: built.Probe, Close: closeProvider}, nil + return node.GenerationProvider{Generation: config.Generation, SpecificationDigest: built.SpecificationDigest, Provider: built.Provider, Probe: built.Probe, Quiescent: built.Quiescent, Close: closeProvider}, nil } type generationJournal struct { diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index a297e4e58..f45a3a2f9 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -180,10 +180,10 @@ func run(config processconfig.Config) error { }) defer managedNodes.close() var managed *execution.RuntimeProvider - observationSources := map[string]runtimeobs.SourceResolver{} + var observationSource func(context.Context) (runtimeobs.Source, string, error) if managedNodes != nil { managed = managedNodes.runtime - observationSources[managed.InstallationID] = managedNodes.setup + observationSource = managedNodes.setup.observationSource } observationResolver, err := deployment.NewObservationResolver(sessionStore, deploymentStore) if err != nil { @@ -193,7 +193,7 @@ func run(config processconfig.Config) error { if err != nil { return err } - observationService, err := runtimeobs.NewService(observationResolver, observationSources, history.Options...) + observationService, err := runtimeobs.NewService(observationResolver, observationSource, history.Options...) if err != nil { if history.Exporter != nil { closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index 8c39850fa..2f7c99b65 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -15,11 +15,7 @@ func (p *generationRouter) Initial(ctx context.Context, r sandbox.Reference) (sa return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.Compute{}, err - } - return cp.Initial(ctx, r) + return v.Initial(ctx, r) } func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, g uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { if err := providercontract.Require(p, "NewCompute"); err != nil { @@ -30,11 +26,7 @@ func (p *generationRouter) NewCompute(ctx context.Context, r sandbox.Reference, return sandbox.Compute{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.Compute{}, err - } - return cp.NewCompute(ctx, r, g, snapshot) + return v.NewCompute(ctx, r, g, snapshot) } func (p *generationRouter) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := providercontract.Require(p, "GetCompute"); err != nil { @@ -45,11 +37,7 @@ func (p *generationRouter) GetCompute(ctx context.Context, r sandbox.Reference, return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.ComputeState{}, err - } - return cp.GetCompute(ctx, r, c) + return v.GetCompute(ctx, r, c) } func (p *generationRouter) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { if err := providercontract.Require(p, "Suspend"); err != nil { @@ -60,11 +48,7 @@ func (p *generationRouter) Suspend(ctx context.Context, q sandbox.SuspendRequest return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.ComputeState{}, err - } - return cp.Suspend(ctx, q) + return v.Suspend(ctx, q) } func (p *generationRouter) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { if err := providercontract.Require(p, "Resume"); err != nil { @@ -75,11 +59,7 @@ func (p *generationRouter) Resume(ctx context.Context, q sandbox.ResumeRequest) return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.ComputeState{}, err - } - return cp.Resume(ctx, q) + return v.Resume(ctx, q) } func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { if err := providercontract.Require(p, "KillCompute"); err != nil { @@ -90,11 +70,7 @@ func (p *generationRouter) KillCompute(ctx context.Context, r sandbox.Reference, return err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return err - } - return cp.KillCompute(ctx, r, c) + return v.KillCompute(ctx, r, c) } func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Reference, snapshot sandbox.SnapshotIdentity) error { if err := providercontract.Require(p, "DeleteSnapshot"); err != nil { @@ -105,11 +81,7 @@ func (p *generationRouter) DeleteSnapshot(ctx context.Context, r sandbox.Referen return err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return err - } - return cp.DeleteSnapshot(ctx, r, snapshot) + return v.DeleteSnapshot(ctx, r, snapshot) } func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { if err := providercontract.Require(p, "RunCommandCompute"); err != nil { @@ -120,11 +92,7 @@ func (p *generationRouter) RunCommandCompute(ctx context.Context, r sandbox.Refe return sandbox.CommandResult{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.CommandResult{}, err - } - return cp.RunCommandCompute(ctx, r, c, command) + return v.RunCommandCompute(ctx, r, c, command) } func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { if err := providercontract.Require(p, "ResumeCompute"); err != nil { @@ -135,15 +103,5 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc return sandbox.ComputeState{}, err } defer done() - cp, err := sandbox.Checkpoint(v) - if err != nil { - return sandbox.ComputeState{}, err - } - return cp.ResumeCompute(ctx, r, c) -} -func (*generationRouter) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "generation_router_does_not_discover_configuration"} -} -func (*generationRouter) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "generation_router_does_not_verify_configuration"} + return v.ResumeCompute(ctx, r, c) } diff --git a/services/core/cmd/server/managed_generations.go b/services/core/cmd/server/managed_generations.go index 3a349f6ed..e509ec494 100644 --- a/services/core/cmd/server/managed_generations.go +++ b/services/core/cmd/server/managed_generations.go @@ -18,9 +18,8 @@ import ( // immutable specification and current credential. There is no mutable provider // map to unload and no current-generation fallback for a missing historical row. type generationRouter struct { - setup *managedSetup - operations providercontract.Operations - providerType string + setup *managedSetup + operations providercontract.Operations } func (p *generationRouter) route(ctx context.Context, r sandbox.Reference) (sandbox.SandboxProvider, func(), error) { @@ -81,18 +80,10 @@ func (p *generationRouter) RunCommand(ctx context.Context, r sandbox.Reference, return v.RunCommand(ctx, r, c) } -type observedGenerationRouter struct{ *generationRouter } - -func (p *observedGenerationRouter) ObservationProviderType() string { return p.providerType } -func (p *observedGenerationRouter) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} - func (p *generationRouter) ProviderOperations() providercontract.Operations { return maps.Clone(p.operations) } - -func (p *observedGenerationRouter) Observe(ctx context.Context, t runtimeobs.Target) (runtimeobs.Sample, error) { +func (p *generationRouter) Observe(ctx context.Context, t runtimeobs.Target) (runtimeobs.Sample, error) { v, done, err := p.route(ctx, sandbox.Reference{TenantID: t.TenantID, EnvironmentID: t.EnvironmentID, AllocationID: t.Instance.AllocationID}) if err != nil { return runtimeobs.Sample{}, err @@ -101,14 +92,7 @@ func (p *observedGenerationRouter) Observe(ctx context.Context, t runtimeobs.Tar if err := providercontract.Require(v, "Observe"); err != nil { return runtimeobs.Sample{}, err } - source, ok := v.(runtimeobs.Source) - if !ok { - return runtimeobs.Sample{}, providercontract.ErrContract - } - if source.ObservationProviderType() != p.providerType { - return runtimeobs.Sample{}, providercontract.ErrContract - } - return source.Observe(ctx, t) + return v.Observe(ctx, t) } // routeGenerations routes a direct provider's allocations through their own @@ -117,11 +101,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo if setup.Mode == "nodes" { return candidate, nil } - router := &generationRouter{setup: s, providerType: candidate.Config.Provider.(runtimeobs.Source).ObservationProviderType(), operations: candidate.Config.Provider.ProviderOperations()} - router.operations["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "allocations_require_individual_generation_routing"} - router.operations["DiscoverSelection"] = providercontract.Support{State: providercontract.Unsupported, Reason: "generation_router_does_not_discover_configuration"} - router.operations["VerifyCredential"] = providercontract.Support{State: providercontract.Unsupported, Reason: "generation_router_does_not_verify_configuration"} - candidate.Config.Provider = &observedGenerationRouter{router} + candidate.Config.Provider = &generationRouter{setup: s, operations: candidate.Config.Provider.ProviderOperations()} if err := sandbox.ValidateProvider(candidate.Config.Provider); err != nil { return execution.PreparedRuntimeDeployment{}, err } @@ -142,18 +122,7 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo // Public template readability cannot establish which team owns a deployment. verify := func(value deployment.Setup, refs []sandbox.Reference) error { value.InstallationID = setup.InstallationID - provider, err := s.provider(value) - if err != nil { - return err - } - if err := providercontract.Require(provider, "VerifyCredential"); err != nil { - return err - } - verifier, ok := provider.(sandbox.CredentialVerifier) - if !ok { - return sandbox.ErrInvalid - } - return verifier.VerifyCredential(ctx, refs) + return s.registry.VerifyCredential(ctx, s.direct(value), refs) } current, err := s.deployment.Setup(ctx) if err != nil { @@ -227,9 +196,3 @@ func (s *managedSetup) routeGenerations(candidate execution.PreparedRuntimeDeplo } return candidate, nil } - -// A batch can contain allocations from different endpoint generations. Let the -// observation worker route each allocation through its immutable generation. -func (p *observedGenerationRouter) ObserveBatch(_ context.Context, _ []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "allocations_require_individual_generation_routing"} -} diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 0ca908728..e1328ba2b 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -116,20 +116,20 @@ func (s *managedSetup) prepare(ctx context.Context, setup deployment.Setup) (exe if err != nil { return execution.PreparedRuntimeDeployment{}, err } - selection := sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration} - if err := providercontract.Require(candidate.Config.Provider, "DiscoverSelection"); err == nil { - discoverer := candidate.Config.Provider.(sandbox.SelectionDiscoverer) - selection, err = discoverer.DiscoverSelection(ctx, selection) - if err != nil { + adapter, err := s.registry.Lookup(setup.Provider) + if err != nil { + return execution.PreparedRuntimeDeployment{}, err + } + direct := s.direct(setup) + selection := direct.Selection + if adapter.Configuration.Requirements().SelectionDiscovery.State == providercontract.Supported { + if selection, err = s.registry.DiscoverSelection(ctx, direct); err != nil { return execution.PreparedRuntimeDeployment{}, err } setup.Specification, setup.Configuration = selection.DeploymentSpec, selection.Configuration - candidate, err = s.configuration(setup) - if err != nil { + if candidate, err = s.configuration(setup); err != nil { return execution.PreparedRuntimeDeployment{}, err } - } else if !errors.Is(err, providercontract.ErrUnsupported) { - return execution.PreparedRuntimeDeployment{}, err } candidate.Selection = &selection return s.routeGenerations(candidate, setup) @@ -145,32 +145,26 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared if err != nil { return execution.PreparedRuntimeDeployment{}, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) } - selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, + selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: s.runtimeAPI, BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } -func (*managedSetup) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}} -} - -func (s *managedSetup) ResolveObservationSource(ctx context.Context) (runtimeobs.Source, error) { +// observationSource returns the selected Provider and its registered kind for +// Runtime observation. +func (s *managedSetup) observationSource(ctx context.Context) (runtimeobs.Source, string, error) { selected, err := s.load(ctx) if err != nil { - return nil, err + return nil, "", err } if selected == nil { - return nil, runtimeobs.ErrUnavailable - } - source, ok := selected.Provider.(runtimeobs.Source) - if !ok { - return nil, providercontract.ErrContract + return nil, "", runtimeobs.ErrUnavailable } - return source, nil + return selected.Provider, selected.ProviderKind, nil } // provider builds the setup's provider. The setup carries the mode and @@ -180,7 +174,12 @@ func (s *managedSetup) provider(setup deployment.Setup) (sandbox.SandboxProvider if s.hub == nil { return nil, errors.New("sandbox node transport is unavailable") } - return s.hub.GenerationProvider(setup.Provider, setup.Operations, s.deployment.AllocationGeneration), nil + return s.hub.GenerationProvider(setup.Operations, s.deployment.AllocationGeneration), nil } - return s.registry.BuildDirect(providers.DirectConfig{ProcessPaths: s.processPaths, InstallationID: setup.InstallationID, Selection: sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration}, Fence: &s.providerCalls}) + return s.registry.BuildDirect(s.direct(setup)) +} + +// direct is the setup's input to direct-mode construction and setup operations. +func (s *managedSetup) direct(setup deployment.Setup) sandbox.DirectConfig { + return sandbox.DirectConfig{ProcessPaths: s.processPaths, InstallationID: setup.InstallationID, Selection: sandbox.Selection{Provider: setup.Provider, DeploymentSpec: setup.Specification, Configuration: setup.Configuration}, Fence: &s.providerCalls} } diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index d6d2ff663..259ed35cb 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -11,7 +11,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" @@ -177,9 +176,9 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { t.Fatal("preparation published or lost candidate configuration") } committed := *candidate.Config - committed.Generation, committed.AdmissionPaused = 2, true + committed.Generation = 2 candidate.Publish(&committed) - if got := s.selected.Load(); got.Generation != 2 || got.Config.ProviderKind != "microsandbox" || !got.Config.AdmissionPaused { + if got := s.selected.Load(); got.Generation != 2 || got.Config.ProviderKind != "microsandbox" { t.Fatal("commit did not publish the validated selection") } } @@ -261,38 +260,21 @@ func testProviderPaths(t *testing.T, helper, state string) sandbox.ProcessPaths func TestManagedObservationSourceKeepsSelectionAcrossReconfiguration(t *testing.T) { value := deployment.Setup{InstallationID: "installation", Generation: 1} setup := &managedSetup{registry: providers.Builtin(), deployment: &fakeDeploymentSetups{t: t, setup: committedSetup(&value)}, installationID: "installation"} - if source, err := setup.ResolveObservationSource(t.Context()); source != nil || !errors.Is(err, runtimeobs.ErrUnavailable) { + if source, kind, err := setup.observationSource(t.Context()); source != nil || kind != "" || !errors.Is(err, runtimeobs.ErrUnavailable) { t.Fatal("unconfigured setup did not return typed unavailability", source, err) } first := &docker.Provider{} value.Provider, value.Mode, value.Generation = "docker", "nodes", 2 - setup.publish(&execution.RuntimeProvider{Generation: 2, Provider: first}) - source, err := setup.ResolveObservationSource(t.Context()) - if err != nil || source != first { - t.Fatal(source, err) + setup.publish(&execution.RuntimeProvider{Generation: 2, ProviderKind: "docker", Provider: first}) + source, kind, err := setup.observationSource(t.Context()) + if err != nil || source != first || kind != "docker" { + t.Fatal(source, kind, err) } next := µsandbox.Provider{} value.Provider, value.Mode, value.Generation = "microsandbox", "nodes", 3 - setup.publish(&execution.RuntimeProvider{Generation: 3, Provider: next}) - if source.ObservationProviderType() != "docker" { - t.Fatal("in-flight identity changed") - } - selected, err := setup.ResolveObservationSource(t.Context()) - if err != nil || selected != next || selected.ObservationProviderType() != "microsandbox" { - t.Fatal(selected, err) - } -} - -func TestObservationGenerationIdentityMustMatchRoutedAllocation(t *testing.T) { - hub := node.NewHub(node.HubOptions{}) - defer hub.Close() - routed := func(context.Context, sandbox.Reference) (deployment.Setup, error) { - return deployment.Setup{Provider: "docker", Mode: "nodes"}, nil - } - setup := &managedSetup{registry: providers.Builtin(), hub: hub, deployment: &fakeDeploymentSetups{t: t, allocationSetup: routed}, allocations: &fakeGenerationAllocations{t: t}} - source := &observedGenerationRouter{&generationRouter{setup: setup, providerType: "e2b"}} - _, err := source.Observe(t.Context(), runtimeobs.Target{TenantID: "tenant", EnvironmentID: "environment", Instance: runtimeobs.Instance{AllocationID: "allocation"}}) - if !errors.Is(err, providercontract.ErrContract) { - t.Fatal("routed allocation was attributed to another provider", err) + setup.publish(&execution.RuntimeProvider{Generation: 3, ProviderKind: "microsandbox", Provider: next}) + selected, kind, err := setup.observationSource(t.Context()) + if err != nil || selected != next || kind != "microsandbox" { + t.Fatal(selected, kind, err) } } diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index ebd7fadf2..04d6c13b3 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -5,7 +5,6 @@ MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 -MAX_OBSERVATION_REFERENCES = 100 MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 @@ -14,5 +13,5 @@ PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] SDK_VERSION = "2.51.0" diff --git a/services/core/internal/api/admin_runtime.go b/services/core/internal/api/admin_runtime.go index f6b1df723..c11dae6aa 100644 --- a/services/core/internal/api/admin_runtime.go +++ b/services/core/internal/api/admin_runtime.go @@ -37,7 +37,7 @@ type AdminRuntimeObservationList struct { } // @Summary List Runtime observations across managed Projects -// @Description Core key only. Each observation is labelled with its owning Project ID. Uses the existing read-only Runtime sampler, with bounded concurrency and no execution or provisioning. A provider with a batch metrics read, such as E2B, samples the page's running sandboxes in one bounded request. +// @Description Core key only. Each observation is labelled with its owning Project ID. Uses the existing read-only Runtime sampler, with bounded concurrency and no execution or provisioning. // @Tags Core Administration // @Produce json // @Security DeploymentAdminAuth diff --git a/services/core/internal/api/agents.go b/services/core/internal/api/agents.go index 0aa2a0607..0e652877f 100644 --- a/services/core/internal/api/agents.go +++ b/services/core/internal/api/agents.go @@ -30,7 +30,7 @@ func (h *Handler) createAgent(w http.ResponseWriter, r *http.Request) { if !ok { return } - if writeFieldError(w, metadataTypeError(raw)) || writeFieldError(w, validateSavedAgentBody(raw, savedAgentCreate)) { + if writeFieldError(w, validateSavedAgentBody(raw, createAgentParams)) { return } if err := validateSavedCoreInput(raw); err != nil { @@ -38,13 +38,14 @@ func (h *Handler) createAgent(w http.ResponseWriter, r *http.Request) { return } var request v1.CreateAgentRequest - if decodeInputObject(raw, &request, "model", "name", "instructions", "metadata", "multi_agent", "reasoning", "service_tier", "text", "tools", "x_agents_core") != nil { + // The walks bound members and types, not integer ranges. + if json.Unmarshal(raw, &request) != nil { writeError(w, http.StatusBadRequest, "invalid_request", "Request must be a JSON object containing supported fields.") return } command, err := resolveSavedAgent(request) if err != nil { - if !writeFieldError(w, err) { + if !writeStoredDataError(w, r, err) && !writeFieldError(w, err) { writeError(w, http.StatusBadRequest, "unsupported_or_invalid_configuration", err.Error()) } return @@ -83,7 +84,7 @@ func (h *Handler) respondAgentStatus(w http.ResponseWriter, r *http.Request, age func agentResponse(agent agents.Agent) (v1.SavedAgent, error) { var response v1.SavedAgent if err := json.Unmarshal(agent.Configuration, &response.SavedAgentConfiguration); err != nil { - return response, err + return response, &storedDataError{err} } response.ID, response.Object = agent.ID, "agent" response.Metadata = agent.Metadata diff --git a/services/core/internal/api/agents_update.go b/services/core/internal/api/agents_update.go index ab7f9d268..b72a4424e 100644 --- a/services/core/internal/api/agents_update.go +++ b/services/core/internal/api/agents_update.go @@ -17,7 +17,7 @@ func (h *Handler) updateAgent(w http.ResponseWriter, r *http.Request) { } command, err := resolveAgentUpdate(raw) if err != nil { - if !writeFieldError(w, err) { + if !writeStoredDataError(w, r, err) && !writeFieldError(w, err) { writeError(w, http.StatusBadRequest, "unsupported_or_invalid_configuration", err.Error()) } return @@ -33,33 +33,25 @@ func (h *Handler) updateAgent(w http.ResponseWriter, r *http.Request) { // resolveAgentUpdate returns the command without its tenant and Agent. func resolveAgentUpdate(raw []byte) (agents.UpdateCommand, error) { - if err := metadataTypeError(raw); err != nil { - return agents.UpdateCommand{}, err - } - if err := validateSavedAgentBody(raw, savedAgentUpdate); err != nil { + if err := validateSavedAgentBody(raw, updateAgentParams); err != nil { return agents.UpdateCommand{}, err } if err := validateSavedCoreInput(raw); err != nil { return agents.UpdateCommand{}, err } var request v1.UpdateAgentRequest - if decodeInputObject(raw, &request, "model", "name", "instructions", "metadata", "multi_agent", "reasoning", "service_tier", "text", "tools", "x_agents_core") != nil { + // The walks bound members and types, not integer ranges. + if json.Unmarshal(raw, &request) != nil { return agents.UpdateCommand{}, errors.New("Request must be a JSON object containing supported fields.") } - var fields map[string]json.RawMessage - if err := json.Unmarshal(raw, &fields); err != nil { - return agents.UpdateCommand{}, err - } - if _, supplied := fields["model"]; supplied && request.Model == nil { - return agents.UpdateCommand{}, errors.New("model must be a string when supplied.") - } + _, fields := orderedMembers(raw) normalized, err := resolveSavedFields(v1.CreateAgentRequest(request)) if err != nil { return agents.UpdateCommand{}, err } var patch map[string]json.RawMessage if err := json.Unmarshal(normalized.Configuration, &patch); err != nil { - return agents.UpdateCommand{}, err + return agents.UpdateCommand{}, &storedDataError{err} } if _, supplied := fields["x_agents_core"]; supplied && request.XAgentsCore == nil { patch["x_agents_core"] = json.RawMessage(`null`) diff --git a/services/core/internal/api/configuration_validation.go b/services/core/internal/api/configuration_validation.go index 2c768821e..577621a26 100644 --- a/services/core/internal/api/configuration_validation.go +++ b/services/core/internal/api/configuration_validation.go @@ -11,24 +11,22 @@ import ( // Agent configuration protocol validation is owned by Core. It covers saved // Agent create and update bodies and the inline Session agent, and runs before // the configuration parsers (which keep Core's local limits and codes) and -// before harness admission. It walks the raw JSON along the pinned SDK shapes -// (PersistedAgentToolParam/AgentToolParam, AgentTextParam, AgentReasoningParam, -// MultiAgentConfigParam and the service_tier literal) and reports the first -// violation as a fieldError with the JSON path as param and the official -// message forms. Values the pinned types leave open, such as JSON Schemas, -// request metadata, MCP transport members, metadata and x_agents_core, are -// left to their existing parsers. +// before harness admission. It walks the raw JSON along the request shapes that +// scripts/generate-public-api.py projects from the pinned schema +// (official_shapes.gen.go) and reports the first violation as a fieldError with +// the JSON path as param and the official message forms. x_agents_core keeps +// its own parser. type valueKind uint8 const ( - anyValue valueKind = iota // validated by the existing parsers + anyValue valueKind = iota // validated by its own parser stringValue // str booleanValue // bool integerValue // int with an inclusive minimum enumValue // Literal[...] of strings arrayValue // Iterable/SequenceNotStr of items - mapValue // Dict[str, object] + mapValue // Dict[str, object], or Dict[str, str] with items openObject // an object whose members are validated elsewhere objectValue // TypedDict members unionValue // TypedDicts selected by their "type" member @@ -42,7 +40,7 @@ type shape struct { values []string // enum values, or union types in the pinned order members []member // objectValue members variants map[string][]member // unionValue members other than "type" - items *shape // arrayValue items + items *shape // arrayValue items, or mapValue string values } type member struct { @@ -50,73 +48,7 @@ type member struct { shape } -var ( - requiredString = shape{kind: stringValue, required: true} - nullableString = shape{kind: stringValue, nullable: true} - stringList = shape{kind: arrayValue, nullable: true, items: &shape{kind: stringValue}} - - agentTools = shape{kind: arrayValue, nullable: true, items: &shape{kind: unionValue, - values: []string{"function", "tool_search", "programmatic_tool_calling", "mcp", "web_search"}, - variants: map[string][]member{ - "function": { - {"description", requiredString}, {"name", requiredString}, - {"parameters", shape{kind: mapValue, required: true}}, {"defer_loading", shape{kind: booleanValue}}, - }, - "tool_search": nil, - "programmatic_tool_calling": {{"enabled", shape{kind: booleanValue}}}, - "mcp": { - {"server_label", requiredString}, {"transport", shape{kind: openObject, required: true}}, - {"allowed_tools", stringList}, - {"connection_origin", shape{kind: enumValue, nullable: true, values: []string{"service", "environment"}}}, - {"credential_id", nullableString}, {"request_metadata", shape{kind: mapValue, nullable: true}}, - {"required", shape{kind: booleanValue}}, - }, - "web_search": { - {"allowed_domains", stringList}, - {"context_size", shape{kind: enumValue, nullable: true, values: []string{"low", "medium", "high"}}}, - {"location", shape{kind: objectValue, nullable: true, members: []member{ - {"city", nullableString}, {"country", nullableString}, {"region", nullableString}, {"timezone", nullableString}, - }}}, - {"mode", shape{kind: enumValue, nullable: true, values: []string{"disabled", "cached", "live"}}}, - }, - }}} - agentText = shape{kind: objectValue, nullable: true, members: []member{ - {"format", shape{kind: unionValue, nullable: true, values: []string{"text", "json_schema"}, - variants: map[string][]member{"text": nil, "json_schema": {{"schema", shape{kind: mapValue, required: true}}}}}}, - {"verbosity", shape{kind: enumValue, nullable: true, values: []string{"low", "medium", "high"}}}, - }} - agentReasoning = shape{kind: objectValue, nullable: true, members: []member{ - {"effort", shape{kind: enumValue, nullable: true, values: []string{"none", "minimal", "low", "medium", "high", "xhigh", "max"}}}, - {"summary", shape{kind: enumValue, nullable: true, values: []string{"concise", "detailed", "auto"}}}, - }} - agentMultiAgent = shape{kind: objectValue, nullable: true, members: []member{ - {"enabled", shape{kind: booleanValue, required: true}}, - {"max_concurrent_subagents", shape{kind: integerValue, minimum: 1}}, - }} - - savedAgentCreate = agentShape(true, true) - savedAgentUpdate = agentShape(true, false) - sessionAgent = agentShape(false, false) -) - -// agentShape returns AgentCreateParams, AgentUpdateParams or the Session Agent. -// Saved Agents additionally carry name and metadata; only creation requires a model. -func agentShape(saved, create bool) shape { - members := []member{ - {"model", shape{kind: stringValue, required: create}}, - {"instructions", nullableString}, - {"multi_agent", agentMultiAgent}, - {"reasoning", agentReasoning}, - {"service_tier", shape{kind: enumValue, nullable: true, values: []string{"auto", "default", "flex", "priority", "fast"}}}, - {"text", agentText}, - {"tools", agentTools}, - {"x_agents_core", shape{}}, - } - if saved { - members = append(members, member{"name", nullableString}, member{"metadata", shape{}}) - } - return shape{kind: objectValue, members: members} -} +var requiredString = shape{kind: stringValue, required: true} // validateSavedAgentBody checks a saved Agent create or update body. Malformed // and non-object bodies keep the existing whole-body error. @@ -129,7 +61,7 @@ func validateSavedAgentBody(raw []byte, root shape) error { // validateSessionAgent checks the inline Session agent, whose paths start with agent. func validateSessionAgent(raw json.RawMessage) error { - return validateAgentConfiguration("agent", raw, sessionAgent) + return validateAgentConfiguration("agent", raw, sessionAgentConfigParam) } func validateAgentConfiguration(path string, raw json.RawMessage, root shape) error { @@ -196,7 +128,19 @@ func checkValue(path string, raw json.RawMessage, s shape) error { } case mapValue: if got != "an object" { - return invalidType(path, "an object with string keys and unknown value values", got) + values := "unknown value" + if s.items != nil { + values = "string" + } + return invalidType(path, "an object with string keys and "+values+" values", got) + } + if s.items != nil { + keys, fields := orderedMembers(raw) + for _, key := range keys { + if err := checkValue(joinPath(path, key), fields[key], *s.items); err != nil { + return err + } + } } case openObject: if got != "an object" { diff --git a/services/core/internal/api/configuration_validation_test.go b/services/core/internal/api/configuration_validation_test.go index 2297b7b32..ed4f7fa6c 100644 --- a/services/core/internal/api/configuration_validation_test.go +++ b/services/core/internal/api/configuration_validation_test.go @@ -83,6 +83,7 @@ func TestAgentConfigurationProtocolErrorsUseOfficialFields(t *testing.T) { {"F03", `"tools":[{"type":"function","name":"lookup","description":"Look up a value."}]`, "{p}tools[0].parameters", "Missing required parameter: '{p}tools[0].parameters'."}, {"F04", `"tools":[` + lookupTool("lookup", `,"strict":true`) + `]`, "{p}tools[0].strict", "Unknown parameter: '{p}tools[0].strict'."}, {"F05", `"tools":[{"type":"function","name":"lookup","parameters":{"type":"object"}}]`, "{p}tools[0].description", "Missing required parameter: '{p}tools[0].description'."}, + {"function order", `"tools":[{"type":"function","parameters":{"type":"object"}}]`, "{p}tools[0].name", "Missing required parameter: '{p}tools[0].name'."}, {"U01", `"tools":[{"type":"code_interpreter"}]`, "{p}tools[0].type", "Invalid value: 'code_interpreter'. Supported values are: 'function'" + tools}, {"U02", `"tools":[{"type":"bogus_tool"}]`, "{p}tools[0].type", "Invalid value: 'bogus_tool'. Supported values are: 'function'" + tools}, {"W03", `"tools":[{"type":"web_search","mode":"bogus"}]`, "{p}tools[0].mode", "Invalid value: 'bogus'. Supported values are: 'disabled', 'cached', and 'live'."}, @@ -110,6 +111,10 @@ func TestAgentConfigurationProtocolErrorsUseOfficialFields(t *testing.T) { {"null enabled", `"tools":[{"type":"programmatic_tool_calling","enabled":null}]`, "{p}tools[0].enabled", "Invalid type for '{p}tools[0].enabled': expected a boolean, but got null instead."}, {"mcp origin", `"tools":[{"type":"mcp","server_label":"x","transport":{"type":"http","server_url":"https://example.invalid"},"connection_origin":"bogus"}]`, "{p}tools[0].connection_origin", "Invalid value: 'bogus'. Supported values are: 'service' and 'environment'."}, {"mcp transport", `"tools":[{"type":"mcp","server_label":"x"}]`, "{p}tools[0].transport", "Missing required parameter: '{p}tools[0].transport'."}, + {"stdio cwd", `"tools":[{"type":"mcp","server_label":"x","transport":{"type":"stdio","command":"run"}}]`, "{p}tools[0].transport.cwd", "Missing required parameter: '{p}tools[0].transport.cwd'."}, + {"http url", `"tools":[{"type":"mcp","server_label":"x","transport":{"type":"http"}}]`, "{p}tools[0].transport.server_url", "Missing required parameter: '{p}tools[0].transport.server_url'."}, + {"transport type", `"tools":[{"type":"mcp","server_label":"x","transport":{"type":"ws"}}]`, "{p}tools[0].transport.type", "Invalid value: 'ws'. Supported values are: 'http' and 'stdio'."}, + {"header value", `"tools":[{"type":"mcp","server_label":"x","transport":{"type":"http","server_url":"https://example.invalid","headers":{"h":1}}}]`, "{p}tools[0].transport.headers.h", "Invalid type for '{p}tools[0].transport.headers.h': expected a string, but got an integer instead."}, {"verbosity", `"text":{"verbosity":"verbose"}`, "{p}text.verbosity", "Invalid value: 'verbose'. Supported values are: 'low', 'medium', and 'high'."}, {"text member", `"text":{"unknown":true}`, "{p}text.unknown", "Unknown parameter: '{p}text.unknown'."}, {"format type", `"text":{"format":{}}`, "{p}text.format.type", "Missing required parameter: '{p}text.format.type'."}, @@ -193,6 +198,9 @@ func TestSessionAgentProtocolErrors(t *testing.T) { } for _, path := range []string{"/v1/agents", "/v1/agents/" + uuid.NewString()} { assertConfigurationError(t, credentialRequest(h, http.MethodPost, path, `{"model":4}`), "invalid_request_error", param("model"), "Invalid type for 'model': expected a string, but got an integer instead.") + assertConfigurationError(t, credentialRequest(h, http.MethodPost, path, `{"model":"m","metadata":5}`), "invalid_request_error", param("metadata"), "Invalid type for 'metadata': expected an object with string keys and string values, but got an integer instead.") + // Inline authorization is a Session-only transport member. + assertConfigurationError(t, credentialRequest(h, http.MethodPost, path, `{"model":"m","tools":[{"type":"mcp","server_label":"x","transport":{"type":"http","server_url":"https://example.invalid","authorization":"x"}}]}`), "invalid_request_error", param("tools[0].transport.authorization"), "Unknown parameter: 'tools[0].transport.authorization'.") assertConfigurationError(t, credentialRequest(h, http.MethodPost, path, `{"model":"m","model":"n"}`), "invalid_request_error", nil, "Invalid body: duplicate JSON key 'model' at 'model'. Duplicate JSON keys are not supported.") } // Saved Agent creation requires a model; updates and Session overrides do not. @@ -267,6 +275,9 @@ func TestAgentConfigurationLocalLimitsKeepCodes(t *testing.T) { for _, op := range configurationOperations()[:2] { assertConfigurationError(t, credentialRequest(h, http.MethodPost, op.path, op.body(`"multi_agent":{"enabled":true,"max_concurrent_subagents":4294967296}`)), "unsupported_or_invalid_configuration", nil, "max_concurrent_subagents must be an integer from 1 to 4294967295.") } + for _, op := range configurationOperations()[:3] { + assertConfigurationError(t, credentialRequest(h, http.MethodPost, op.path, op.body(`"tools":[{"type":"mcp","server_label":"x","transport":{"type":"stdio","command":"run","cwd":"/"}}]`)), "unsupported_or_invalid_configuration", nil, "MCP currently supports HTTP transport only.") + } if s.writes != 0 { t.Fatal("rejected configuration reached storage") } diff --git a/services/core/internal/api/core_error_catalog_test.go b/services/core/internal/api/core_error_catalog_test.go index 2d81518ab..42d36dee5 100644 --- a/services/core/internal/api/core_error_catalog_test.go +++ b/services/core/internal/api/core_error_catalog_test.go @@ -24,8 +24,6 @@ var nonAdministrationCodes = []string{ "environment_input_cancelled", "environment_input_expired", "model_provider_required", "sandbox_nodes_preparing", "turn_conflict", // Machine routes for nodes and native installers. "installation_authorization_invalid", "installation_unavailable", "invalid_node_credential", "sandbox_node_address_mismatch", - // Removal of the file-managed local node, which the process deployment path owns. - "runtime_local_node_configured", } // The shared catalog lists every code an administration caller (/core/v1 or diff --git a/services/core/internal/api/disabled_tools.go b/services/core/internal/api/disabled_tools.go index b52f3d9a3..884f8092d 100644 --- a/services/core/internal/api/disabled_tools.go +++ b/services/core/internal/api/disabled_tools.go @@ -5,31 +5,28 @@ import ( "errors" ) -func resolveProgrammaticTool(raw json.RawMessage) (json.RawMessage, error) { - var input struct { - Type string `json:"type"` - Enabled json.RawMessage `json:"enabled"` - } - if decodeInputObject(raw, &input, "type", "enabled") != nil { - return nil, errors.New("Invalid programmatic_tool_calling fields.") - } - enabled, err := optionalBoolean(input.Enabled, true) - if err != nil { - return nil, errors.New("programmatic_tool_calling.enabled must be a boolean.") - } - return json.Marshal(struct { +// Tool resolvers read tools whose pinned shape was checked at the /v1 boundary: +// request tools, or a saved Agent's tools that resolveSavedTools stored. A +// decode failure is Core's own fault and is reported as a service error. +func resolveProgrammaticTool(raw json.RawMessage) (json.RawMessage, bool, error) { + input := struct { Type string `json:"type"` Enabled bool `json:"enabled"` - }{input.Type, enabled}) + }{Enabled: true} + if err := json.Unmarshal(raw, &input); err != nil { + return nil, false, &storedDataError{err} + } + value, err := json.Marshal(input) + return value, input.Enabled, err } // webSearchTool is the resolved web_search projection. A present location // projects all four keys; null and empty allowed_domains stay distinct. type webSearchTool struct { - Type string `json:"type"` - Mode *string `json:"mode"` - ContextSize *string `json:"context_size"` - AllowedDomains []*string `json:"allowed_domains"` + Type string `json:"type"` + Mode *string `json:"mode"` + ContextSize *string `json:"context_size"` + AllowedDomains []string `json:"allowed_domains"` Location *struct { City *string `json:"city"` Country *string `json:"country"` @@ -38,27 +35,12 @@ type webSearchTool struct { } `json:"location"` } -func decodeWebSearch(raw json.RawMessage) (webSearchTool, bool) { - var tool webSearchTool - return tool, decodeInputObject(raw, &tool, "type", "mode", "context_size", "allowed_domains", "location") == nil -} - // Optional settings are resource data in every mode; they never enable execution. func (tool webSearchTool) resolveSettings() (json.RawMessage, error) { if tool.ContextSize == nil { value := "medium" tool.ContextSize = &value } - for _, domain := range tool.AllowedDomains { - if domain == nil { - return nil, errors.New("web_search.allowed_domains must contain strings.") - } - } - switch *tool.ContextSize { - case "low", "medium", "high": - default: - return nil, errors.New("Invalid web_search.context_size.") - } return json.Marshal(tool) } @@ -66,26 +48,24 @@ func (tool webSearchTool) resolveSettings() (json.RawMessage, error) { // or null mode is saved as live. Session admission still qualifies only disabled // search (resolveDisabledWebSearch), so saving never enables execution. func resolveSavedWebSearch(raw json.RawMessage) (json.RawMessage, error) { - tool, ok := decodeWebSearch(raw) - if !ok { - return nil, errors.New("Invalid web_search fields.") + var tool webSearchTool + if err := json.Unmarshal(raw, &tool); err != nil { + return nil, &storedDataError{err} } if tool.Mode == nil { value := "live" tool.Mode = &value } - switch *tool.Mode { - case "disabled", "cached", "live": - default: - return nil, errors.New("web_search.mode must be disabled, cached or live.") - } return tool.resolveSettings() } // Only disabled search is qualified for execution. func resolveDisabledWebSearch(raw json.RawMessage) (json.RawMessage, error) { - tool, ok := decodeWebSearch(raw) - if !ok || tool.Mode == nil || *tool.Mode != "disabled" { + var tool webSearchTool + if err := json.Unmarshal(raw, &tool); err != nil { + return nil, &storedDataError{err} + } + if tool.Mode == nil || *tool.Mode != "disabled" { return nil, errors.New("Only disabled web_search is qualified for execution.") } return tool.resolveSettings() diff --git a/services/core/internal/api/errors.go b/services/core/internal/api/errors.go index 676d51ee0..2e92da06d 100644 --- a/services/core/internal/api/errors.go +++ b/services/core/internal/api/errors.go @@ -105,6 +105,28 @@ func writeInternalError(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusInternalServerError, "internal_error", "The operation could not be completed.") } +// storedDataError carries a failure of Core's own stored or resolved +// configuration on the Agent and Session paths: a saved configuration or tool +// that Core resolved or stored and that does not decode, or a storage or +// decryption failure while reading the deployment default model provider. It +// is never the client's input, so it is never echoed in a response. +type storedDataError struct{ err error } + +func (e *storedDataError) Error() string { return e.err.Error() } +func (e *storedDataError) Unwrap() error { return e.err } + +// writeStoredDataError reports a storedDataError as a service error and returns +// false for any other error. The underlying failure is logged for diagnosis. +func writeStoredDataError(w http.ResponseWriter, r *http.Request, err error) bool { + var stored *storedDataError + if !errors.As(err, &stored) { + return false + } + log.Ctx(r.Context()).Error("oac-core stored data failed", "error", stored.err) + writeModelConfigurationError(w, r, stored.err) + return true +} + // writeTextValueError reports request text that PostgreSQL cannot store and // returns false for any other error. It is a documented local limit: text and // jsonb cannot store U+0000, and text parameters, including query filters, diff --git a/services/core/internal/api/errors_agents.go b/services/core/internal/api/errors_agents.go index 211e6c3fe..4c6391793 100644 --- a/services/core/internal/api/errors_agents.go +++ b/services/core/internal/api/errors_agents.go @@ -11,7 +11,7 @@ import ( // writeAgentsError reports an error of the Agent operations. func writeAgentsError(w http.ResponseWriter, r *http.Request, err error) { - if writeTextValueError(w, r, err) || writeAuditSourceError(w, r, err) || writeCredentialUnavailableError(w, r, err) { + if writeStoredDataError(w, r, err) || writeTextValueError(w, r, err) || writeAuditSourceError(w, r, err) || writeCredentialUnavailableError(w, r, err) { return } var provider *v1.ModelProviderError diff --git a/services/core/internal/api/errors_deployment.go b/services/core/internal/api/errors_deployment.go index 70435254a..0da81d951 100644 --- a/services/core/internal/api/errors_deployment.go +++ b/services/core/internal/api/errors_deployment.go @@ -102,8 +102,6 @@ func writeSandboxError(w http.ResponseWriter, err error) bool { writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") case errors.Is(err, deployment.ErrNodeInUse): writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") - case errors.Is(err, deployment.ErrLocalNodeConfigured): - writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") case errors.Is(err, placement.ErrNodesPreparing): writeError(w, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime.") case errors.Is(err, placement.ErrNodeUnavailable): diff --git a/services/core/internal/api/errors_test.go b/services/core/internal/api/errors_test.go index 9f139d003..25cc78eb9 100644 --- a/services/core/internal/api/errors_test.go +++ b/services/core/internal/api/errors_test.go @@ -138,7 +138,6 @@ func TestConflictErrorsUseConflictType(t *testing.T) { for err, code := range map[error]string{ deployment.ErrConflict: "sandbox_deployment_conflict", deployment.ErrNodeInUse: "runtime_node_in_use", - deployment.ErrLocalNodeConfigured: "runtime_local_node_configured", deployment.ErrNodeAddressMismatch: "sandbox_node_address_mismatch", sessions.ErrEnvironmentUnavailable: "environment_unavailable", execution.ErrEnvironmentInputExpired: "environment_input_expired", diff --git a/services/core/internal/api/function_configuration.go b/services/core/internal/api/function_configuration.go index 6d013317c..0674a43f5 100644 --- a/services/core/internal/api/function_configuration.go +++ b/services/core/internal/api/function_configuration.go @@ -1,7 +1,6 @@ package api import ( - "bytes" "encoding/json" "errors" "strings" @@ -16,51 +15,28 @@ func resolveFunctions(input []v1.FunctionToolInput) ([]json.RawMessage, error) { } names := make(map[string]bool, len(input)) for _, tool := range input { - value, _, err := resolveFunction(tool) + if strings.TrimSpace(tool.Name) == "" || len(tool.Name) > 512 || names[tool.Name] { + return nil, errors.New("Function names must be nonempty, unique and at most 512 bytes.") + } + value, err := resolveFunction(tool) if err != nil { return nil, err } - if strings.TrimSpace(*tool.Name) == "" || len(*tool.Name) > 512 || names[*tool.Name] { - return nil, errors.New("Function names must be nonempty, unique and at most 512 bytes.") - } - - names[*tool.Name] = true + names[tool.Name] = true tools = append(tools, value) } return tools, nil } -// resolveFunction validates the persisted wire shape. Execution admission may -// impose additional restrictions, without narrowing the reusable resource. -func resolveFunction(tool v1.FunctionToolInput) (json.RawMessage, bool, error) { - if tool.Type != "function" || tool.Name == nil || tool.Description == nil { - return nil, false, errors.New("Function tools require type=function, name and description.") - } - var schema map[string]json.RawMessage - if json.Unmarshal(tool.Parameters, &schema) != nil || schema == nil { - return nil, false, errors.New("Function parameters must be a JSON Schema object.") - } - deferred, err := optionalBoolean(tool.DeferLoading, false) - if err != nil { - return nil, false, errors.New("defer_loading must be a boolean when supplied.") - } - value, err := json.Marshal(struct { +// resolveFunction canonicalizes a function whose pinned shape was checked at +// the /v1 boundary. Execution admission may impose additional restrictions, +// without narrowing the reusable resource. +func resolveFunction(tool v1.FunctionToolInput) (json.RawMessage, error) { + return json.Marshal(struct { Type string `json:"type"` Name string `json:"name"` Description string `json:"description"` Parameters json.RawMessage `json:"parameters"` DeferLoading bool `json:"defer_loading"` - }{"function", *tool.Name, *tool.Description, tool.Parameters, deferred}) - return value, deferred, err -} - -func optionalBoolean(raw json.RawMessage, fallback bool) (bool, error) { - if len(raw) == 0 { - return fallback, nil - } - var value bool - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) || json.Unmarshal(raw, &value) != nil { - return false, errors.New("Expected a boolean.") - } - return value, nil + }{"function", tool.Name, tool.Description, tool.Parameters, tool.DeferLoading != nil && *tool.DeferLoading}) } diff --git a/services/core/internal/api/handler.go b/services/core/internal/api/handler.go index 2e358785e..33d76be58 100644 --- a/services/core/internal/api/handler.go +++ b/services/core/internal/api/handler.go @@ -232,12 +232,10 @@ func (h *Handler) createSession(w http.ResponseWriter, r *http.Request) { return } var required *modelProviderRequiredError - var defaults *modelProviderDefaultsError switch { case errors.As(err, &required): writeError(w, http.StatusBadRequest, "model_provider_required", required.message, "x_agents_core.model_provider") - case errors.As(err, &defaults): - writeModelConfigurationError(w, r, defaults.err) + case writeStoredDataError(w, r, err): case !writeFieldError(w, err): writeError(w, http.StatusBadRequest, "unsupported_or_invalid_configuration", err.Error()) } diff --git a/services/core/internal/api/mcp_configuration.go b/services/core/internal/api/mcp_configuration.go index e856bdb55..6af603b52 100644 --- a/services/core/internal/api/mcp_configuration.go +++ b/services/core/internal/api/mcp_configuration.go @@ -11,92 +11,63 @@ import ( const mcpHTTPOnly = "MCP currently supports HTTP transport only." +// resolveMCPTool canonicalizes a declaration whose pinned shape was checked at +// decode. Core executes HTTP servers only and keeps its local limits. func resolveMCPTool(raw json.RawMessage, saved bool) (json.RawMessage, error) { var input v1.MCPToolInput - if decodeInputObject(raw, &input, "type", "server_label", "transport", "allowed_tools", "connection_origin", "credential_id", "request_metadata", "required") != nil { - return nil, errors.New("Invalid MCP tool fields.") + var transport struct { + Type string `json:"type"` + ServerURL string `json:"server_url"` + Authorization *string `json:"authorization"` + Headers map[string]string `json:"headers"` + } + if err := errors.Join(json.Unmarshal(raw, &input), json.Unmarshal(input.Transport, &transport)); err != nil { + return nil, &storedDataError{err} } - if input.Type != "mcp" || input.ServerLabel == nil || strings.TrimSpace(*input.ServerLabel) == "" { + if strings.TrimSpace(input.ServerLabel) == "" { return nil, errors.New("MCP tools require type=mcp and a nonempty server_label.") } + if transport.Type != "http" { + return nil, errors.New(mcpHTTPOnly) + } // The official service saves an omitted or null origin on an HTTP server as // "service" (MV-01). Defaulting it here makes the stored and frozen - // configuration identical to an explicit declaration. Other transports are - // unsupported with any origin. - if input.ConnectionOrigin == nil { - if !mcpHTTPTransport(input.Transport) { - return nil, errors.New(mcpHTTPOnly) - } - service := "service" - input.ConnectionOrigin = &service - } - if *input.ConnectionOrigin != "service" && *input.ConnectionOrigin != "environment" { - return nil, errors.New("MCP connection_origin must be service or environment.") + // configuration identical to an explicit declaration. + origin := "service" + if input.ConnectionOrigin != nil { + origin = *input.ConnectionOrigin } if input.CredentialID != nil && *input.CredentialID == "" { return nil, errors.New("MCP credential_id must be null or a nonempty string.") } - required, err := optionalBoolean(input.Required, false) - if err != nil { - return nil, errors.New("MCP required must be a boolean.") - } - if !emptyMCPObject(input.RequestMetadata) { + if len(input.RequestMetadata) != 0 { return nil, errors.New("Nonempty MCP request_metadata is not supported yet.") } - var transport struct { - Type string `json:"type"` - ServerURL *string `json:"server_url"` - Headers json.RawMessage `json:"headers"` - } - if decodeInputObject(input.Transport, &transport, "type", "server_url", "headers") != nil || transport.Type != "http" || transport.ServerURL == nil { - return nil, errors.New(mcpHTTPOnly) - } - u, err := url.Parse(*transport.ServerURL) + u, err := url.Parse(transport.ServerURL) if err != nil || u.Hostname() == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.Fragment != "" || u.RawQuery != "" || u.ForceQuery { return nil, errors.New("MCP server_url must be an absolute HTTP(S) URL without credentials, query or fragment.") } - if !emptyMCPObject(transport.Headers) { + if transport.Authorization != nil { + return nil, errors.New("Inline MCP authorization is not supported yet.") + } + if len(transport.Headers) != 0 { return nil, errors.New("Nonempty MCP headers are not supported yet.") } var allowed *[]string - if len(input.AllowedTools) != 0 { - var values *[]*string - if json.Unmarshal(input.AllowedTools, &values) != nil { - return nil, errors.New("MCP allowed_tools must be null or an array of tool names.") - } - if values != nil { - names := make([]string, 0, len(*values)) - for _, name := range *values { - if name == nil || *name == "" { - return nil, errors.New("MCP allowed_tools requires nonempty string names.") - } - names = append(names, *name) + if input.AllowedTools != nil { + for _, name := range input.AllowedTools { + if name == "" { + return nil, errors.New("MCP allowed_tools requires nonempty string names.") } - allowed = &names } + allowed = &input.AllowedTools } - tool := v1.MCPTool{Type: "mcp", ServerLabel: *input.ServerLabel, - Transport: v1.MCPHTTPTransport{Type: "http", ServerURL: *transport.ServerURL}, - AllowedTools: allowed, Required: required, ConnectionOrigin: *input.ConnectionOrigin, CredentialID: input.CredentialID, RequestMetadata: map[string]json.RawMessage{}} + tool := v1.MCPTool{Type: "mcp", ServerLabel: input.ServerLabel, + Transport: v1.MCPHTTPTransport{Type: "http", ServerURL: transport.ServerURL}, + AllowedTools: allowed, Required: input.Required != nil && *input.Required, ConnectionOrigin: origin, CredentialID: input.CredentialID, RequestMetadata: map[string]json.RawMessage{}} if saved { headers := map[string]string{} tool.Transport.Headers = &headers } return json.Marshal(tool) } - -// mcpHTTPTransport reports a transport object whose exact "type" member is -// "http". The complete transport is validated afterwards. -func mcpHTTPTransport(raw json.RawMessage) bool { - var fields map[string]json.RawMessage - var kind string - return json.Unmarshal(raw, &fields) == nil && json.Unmarshal(fields["type"], &kind) == nil && kind == "http" -} - -func emptyMCPObject(raw json.RawMessage) bool { - if len(raw) == 0 { - return true - } - var value map[string]json.RawMessage - return json.Unmarshal(raw, &value) == nil && len(value) == 0 -} diff --git a/services/core/internal/api/mcp_configuration_test.go b/services/core/internal/api/mcp_configuration_test.go index 49ecc87e7..f0013cafb 100644 --- a/services/core/internal/api/mcp_configuration_test.go +++ b/services/core/internal/api/mcp_configuration_test.go @@ -70,7 +70,7 @@ func TestMCPOmittedOriginIsService(t *testing.T) { } // Other transports report the transport restriction with any origin. for _, origin := range []string{"", `,"connection_origin":null`, `,"connection_origin":"service"`} { - input := `{"type":"mcp","server_label":"records"` + origin + `,"transport":{"type":"stdio","command":"run"}}` + input := `{"type":"mcp","server_label":"records"` + origin + `,"transport":{"type":"stdio","command":"run","cwd":"/"}}` if _, err := resolveMCPTool(json.RawMessage(input), true); err == nil || err.Error() != "MCP currently supports HTTP transport only." { t.Fatalf("stdio origin %q: %v", origin, err) } @@ -85,6 +85,7 @@ func TestMCPAllowedToolsAndOptionalFields(t *testing.T) { } input["allowed_tools"] = json.RawMessage(allowed) input["credential_id"], input["request_metadata"], input["required"] = json.RawMessage("null"), json.RawMessage("null"), json.RawMessage("false") + input["transport"] = json.RawMessage(`{"type":"http","server_url":"https://mcp.example.test/tools","authorization":null,"headers":null}`) encoded, _ := json.Marshal(input) resolved, err := resolveMCPTool(encoded, false) if err != nil { @@ -97,24 +98,18 @@ func TestMCPAllowedToolsAndOptionalFields(t *testing.T) { } } +// The pinned shape is checked at decode; these inputs match it. func TestMCPUnsupportedInputsAreSecretSafe(t *testing.T) { for name, replacement := range map[string]map[string]json.RawMessage{ - "unknown origin": {"connection_origin": json.RawMessage(`"unknown"`)}, - "stdio origin missing": {"connection_origin": nil, "transport": json.RawMessage(`{"type":"stdio","command":"private-marker"}`)}, - "stdio origin null": {"connection_origin": json.RawMessage("null"), "transport": json.RawMessage(`{"type":"stdio","command":"private-marker"}`)}, - "origin missing, case": {"connection_origin": nil, "transport": json.RawMessage(`{"Type":"http","server_url":"https://mcp.example.test"}`)}, - "required type": {"required": json.RawMessage(`"true"`)}, - "required null": {"required": json.RawMessage("null")}, + "stdio origin missing": {"connection_origin": nil, "transport": json.RawMessage(`{"type":"stdio","command":"private-marker","cwd":"/"}`)}, + "stdio origin null": {"connection_origin": json.RawMessage("null"), "transport": json.RawMessage(`{"type":"stdio","command":"private-marker","cwd":"/"}`)}, "empty credential": {"credential_id": json.RawMessage(`""`)}, - "credential type": {"credential_id": json.RawMessage(`3`)}, "metadata": {"request_metadata": json.RawMessage(`{"private-marker":"value"}`)}, - "null tool name": {"allowed_tools": json.RawMessage(`[null]`)}, - "wrong allow-list": {"allowed_tools": json.RawMessage(`"lookup"`)}, "inline authorization": {"transport": json.RawMessage(`{"type":"http","server_url":"https://mcp.example.test","authorization":"private-marker"}`)}, "headers": {"transport": json.RawMessage(`{"type":"http","server_url":"https://mcp.example.test","headers":{"Authorization":"private-marker"}}`)}, "URL credentials": {"transport": json.RawMessage(`{"type":"http","server_url":"https://private-marker@mcp.example.test"}`)}, "URL query": {"transport": json.RawMessage(`{"type":"http","server_url":"https://mcp.example.test/?token=private-marker"}`)}, - "stdio": {"transport": json.RawMessage(`{"type":"stdio","command":"private-marker"}`)}, + "stdio": {"transport": json.RawMessage(`{"type":"stdio","command":"private-marker","cwd":"/"}`)}, } { t.Run(name, func(t *testing.T) { var input map[string]json.RawMessage diff --git a/services/core/internal/api/official_shapes.gen.go b/services/core/internal/api/official_shapes.gen.go new file mode 100644 index 000000000..b4ba45f34 --- /dev/null +++ b/services/core/internal/api/official_shapes.gen.go @@ -0,0 +1,210 @@ +// Code generated by scripts/generate-public-api.py; DO NOT EDIT. +package api + +// Pinned request shapes; x_agents_core is checked by its own parser. +var ( + createAgentParams = shape{kind: objectValue, members: []member{ + {"metadata", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + {"name", shape{kind: stringValue, nullable: true}}, + {"model", shape{kind: stringValue, required: true}}, + {"reasoning", shape{kind: objectValue, members: []member{ + {"effort", shape{kind: enumValue, values: []string{"none", "minimal", "low", "medium", "high", "xhigh", "max"}, nullable: true}}, + {"summary", shape{kind: enumValue, values: []string{"concise", "detailed", "auto"}, nullable: true}}, + }, nullable: true}}, + {"text", shape{kind: objectValue, members: []member{ + {"format", shape{kind: unionValue, values: []string{"text", "json_schema"}, variants: map[string][]member{ + "text": {}, + "json_schema": { + {"schema", shape{kind: mapValue, required: true}}, + }, + }, nullable: true}}, + {"verbosity", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + }, nullable: true}}, + {"service_tier", shape{kind: enumValue, values: []string{"auto", "default", "flex", "priority", "fast"}, nullable: true}}, + {"instructions", shape{kind: stringValue, nullable: true}}, + {"tools", shape{kind: arrayValue, items: &shape{kind: unionValue, values: []string{"function", "tool_search", "programmatic_tool_calling", "mcp", "web_search"}, variants: map[string][]member{ + "function": { + {"name", shape{kind: stringValue, required: true}}, + {"description", shape{kind: stringValue, required: true}}, + {"parameters", shape{kind: mapValue, required: true}}, + {"defer_loading", shape{kind: booleanValue}}, + }, + "tool_search": {}, + "programmatic_tool_calling": { + {"enabled", shape{kind: booleanValue}}, + }, + "mcp": { + {"server_label", shape{kind: stringValue, required: true}}, + {"credential_id", shape{kind: stringValue, nullable: true}}, + {"transport", shape{kind: unionValue, values: []string{"http", "stdio"}, variants: map[string][]member{ + "http": { + {"server_url", shape{kind: stringValue, required: true}}, + {"headers", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + "stdio": { + {"command", shape{kind: stringValue, required: true}}, + {"args", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"cwd", shape{kind: stringValue, required: true}}, + {"env_vars", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + }, required: true}}, + {"request_metadata", shape{kind: mapValue, nullable: true}}, + {"allowed_tools", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"required", shape{kind: booleanValue}}, + {"connection_origin", shape{kind: enumValue, values: []string{"service", "environment"}, nullable: true}}, + }, + "web_search": { + {"mode", shape{kind: enumValue, values: []string{"disabled", "cached", "live"}, nullable: true}}, + {"context_size", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + {"allowed_domains", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"location", shape{kind: objectValue, members: []member{ + {"country", shape{kind: stringValue, nullable: true}}, + {"region", shape{kind: stringValue, nullable: true}}, + {"city", shape{kind: stringValue, nullable: true}}, + {"timezone", shape{kind: stringValue, nullable: true}}, + }, nullable: true}}, + }, + }}, nullable: true}}, + {"multi_agent", shape{kind: objectValue, members: []member{ + {"enabled", shape{kind: booleanValue, required: true}}, + {"max_concurrent_subagents", shape{kind: integerValue, minimum: 1}}, + }, nullable: true}}, + {"x_agents_core", shape{}}, + }} + updateAgentParams = shape{kind: objectValue, members: []member{ + {"model", shape{kind: stringValue}}, + {"reasoning", shape{kind: objectValue, members: []member{ + {"effort", shape{kind: enumValue, values: []string{"none", "minimal", "low", "medium", "high", "xhigh", "max"}, nullable: true}}, + {"summary", shape{kind: enumValue, values: []string{"concise", "detailed", "auto"}, nullable: true}}, + }, nullable: true}}, + {"text", shape{kind: objectValue, members: []member{ + {"format", shape{kind: unionValue, values: []string{"text", "json_schema"}, variants: map[string][]member{ + "text": {}, + "json_schema": { + {"schema", shape{kind: mapValue, required: true}}, + }, + }, nullable: true}}, + {"verbosity", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + }, nullable: true}}, + {"service_tier", shape{kind: enumValue, values: []string{"auto", "default", "flex", "priority", "fast"}, nullable: true}}, + {"instructions", shape{kind: stringValue, nullable: true}}, + {"multi_agent", shape{kind: objectValue, members: []member{ + {"enabled", shape{kind: booleanValue, required: true}}, + {"max_concurrent_subagents", shape{kind: integerValue, minimum: 1}}, + }, nullable: true}}, + {"metadata", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + {"name", shape{kind: stringValue, nullable: true}}, + {"tools", shape{kind: arrayValue, items: &shape{kind: unionValue, values: []string{"function", "tool_search", "programmatic_tool_calling", "mcp", "web_search"}, variants: map[string][]member{ + "function": { + {"name", shape{kind: stringValue, required: true}}, + {"description", shape{kind: stringValue, required: true}}, + {"parameters", shape{kind: mapValue, required: true}}, + {"defer_loading", shape{kind: booleanValue}}, + }, + "tool_search": {}, + "programmatic_tool_calling": { + {"enabled", shape{kind: booleanValue}}, + }, + "mcp": { + {"server_label", shape{kind: stringValue, required: true}}, + {"credential_id", shape{kind: stringValue, nullable: true}}, + {"transport", shape{kind: unionValue, values: []string{"http", "stdio"}, variants: map[string][]member{ + "http": { + {"server_url", shape{kind: stringValue, required: true}}, + {"headers", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + "stdio": { + {"command", shape{kind: stringValue, required: true}}, + {"args", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"cwd", shape{kind: stringValue, required: true}}, + {"env_vars", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + }, required: true}}, + {"request_metadata", shape{kind: mapValue, nullable: true}}, + {"allowed_tools", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"required", shape{kind: booleanValue}}, + {"connection_origin", shape{kind: enumValue, values: []string{"service", "environment"}, nullable: true}}, + }, + "web_search": { + {"mode", shape{kind: enumValue, values: []string{"disabled", "cached", "live"}, nullable: true}}, + {"context_size", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + {"allowed_domains", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"location", shape{kind: objectValue, members: []member{ + {"country", shape{kind: stringValue, nullable: true}}, + {"region", shape{kind: stringValue, nullable: true}}, + {"city", shape{kind: stringValue, nullable: true}}, + {"timezone", shape{kind: stringValue, nullable: true}}, + }, nullable: true}}, + }, + }}, nullable: true}}, + {"x_agents_core", shape{}}, + }} + sessionAgentConfigParam = shape{kind: objectValue, members: []member{ + {"model", shape{kind: stringValue}}, + {"reasoning", shape{kind: objectValue, members: []member{ + {"effort", shape{kind: enumValue, values: []string{"none", "minimal", "low", "medium", "high", "xhigh", "max"}, nullable: true}}, + {"summary", shape{kind: enumValue, values: []string{"concise", "detailed", "auto"}, nullable: true}}, + }, nullable: true}}, + {"text", shape{kind: objectValue, members: []member{ + {"format", shape{kind: unionValue, values: []string{"text", "json_schema"}, variants: map[string][]member{ + "text": {}, + "json_schema": { + {"schema", shape{kind: mapValue, required: true}}, + }, + }, nullable: true}}, + {"verbosity", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + }, nullable: true}}, + {"service_tier", shape{kind: enumValue, values: []string{"auto", "default", "flex", "priority", "fast"}, nullable: true}}, + {"instructions", shape{kind: stringValue, nullable: true}}, + {"multi_agent", shape{kind: objectValue, members: []member{ + {"enabled", shape{kind: booleanValue, required: true}}, + {"max_concurrent_subagents", shape{kind: integerValue, minimum: 1}}, + }, nullable: true}}, + {"tools", shape{kind: arrayValue, items: &shape{kind: unionValue, values: []string{"function", "tool_search", "programmatic_tool_calling", "mcp", "web_search"}, variants: map[string][]member{ + "function": { + {"name", shape{kind: stringValue, required: true}}, + {"description", shape{kind: stringValue, required: true}}, + {"parameters", shape{kind: mapValue, required: true}}, + {"defer_loading", shape{kind: booleanValue}}, + }, + "tool_search": {}, + "programmatic_tool_calling": { + {"enabled", shape{kind: booleanValue}}, + }, + "mcp": { + {"server_label", shape{kind: stringValue, required: true}}, + {"credential_id", shape{kind: stringValue, nullable: true}}, + {"transport", shape{kind: unionValue, values: []string{"http", "stdio"}, variants: map[string][]member{ + "http": { + {"server_url", shape{kind: stringValue, required: true}}, + {"authorization", shape{kind: stringValue, nullable: true}}, + {"headers", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + "stdio": { + {"command", shape{kind: stringValue, required: true}}, + {"args", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"cwd", shape{kind: stringValue, required: true}}, + {"env", shape{kind: mapValue, items: &shape{kind: stringValue}, nullable: true}}, + {"env_vars", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + }, + }, required: true}}, + {"request_metadata", shape{kind: mapValue, nullable: true}}, + {"allowed_tools", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"required", shape{kind: booleanValue}}, + {"connection_origin", shape{kind: enumValue, values: []string{"service", "environment"}, nullable: true}}, + }, + "web_search": { + {"mode", shape{kind: enumValue, values: []string{"disabled", "cached", "live"}, nullable: true}}, + {"context_size", shape{kind: enumValue, values: []string{"low", "medium", "high"}, nullable: true}}, + {"allowed_domains", shape{kind: arrayValue, items: &shape{kind: stringValue}, nullable: true}}, + {"location", shape{kind: objectValue, members: []member{ + {"country", shape{kind: stringValue, nullable: true}}, + {"region", shape{kind: stringValue, nullable: true}}, + {"city", shape{kind: stringValue, nullable: true}}, + {"timezone", shape{kind: stringValue, nullable: true}}, + }, nullable: true}}, + }, + }}, nullable: true}}, + {"x_agents_core", shape{}}, + }} +) diff --git a/services/core/internal/api/placement_errors_test.go b/services/core/internal/api/placement_errors_test.go index 90a8fb360..8aefad2be 100644 --- a/services/core/internal/api/placement_errors_test.go +++ b/services/core/internal/api/placement_errors_test.go @@ -19,7 +19,7 @@ func TestPlacementErrorsKeepTheirResponses(t *testing.T) { code, message string }{ {placement.ErrResetAdmission, http.StatusServiceUnavailable, "sandbox_reset_in_progress", "A sandbox reset is in progress."}, - {fmt.Errorf("%w: sandbox creation is paused for provider maintenance", placement.ErrAdmissionClosed), http.StatusConflict, "environment_unavailable", "The environment is no longer available for new input."}, + {fmt.Errorf("%w: sandbox installation does not match deployment", placement.ErrAdmissionClosed), http.StatusConflict, "environment_unavailable", "The environment is no longer available for new input."}, {placement.ErrPublicURLUnreachable, http.StatusConflict, "sandbox_configuration_error", placement.ErrPublicURLUnreachable.Error()}, {placement.ErrNodesPreparing, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime."}, {placement.ErrNodeUnavailable, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity."}, diff --git a/services/core/internal/api/sandbox_deployment_setup.go b/services/core/internal/api/sandbox_deployment_setup.go index b58295306..5a26510b8 100644 --- a/services/core/internal/api/sandbox_deployment_setup.go +++ b/services/core/internal/api/sandbox_deployment_setup.go @@ -51,7 +51,7 @@ type DeploymentReset interface { } // @Summary Initialize the deployment sandbox provider -// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. +// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth diff --git a/services/core/internal/api/sandbox_manager_test.go b/services/core/internal/api/sandbox_manager_test.go index 7de004b17..6e7959622 100644 --- a/services/core/internal/api/sandbox_manager_test.go +++ b/services/core/internal/api/sandbox_manager_test.go @@ -69,15 +69,6 @@ func TestSandboxEnrollmentDoesNotAcceptProjectAsAdmin(t *testing.T) { } } -func TestSandboxLocalNodeRemovalExplainsDeploymentBinding(t *testing.T) { - request := httptest.NewRequest(http.MethodDelete, "/core/v1/sandbox/nodes/local", nil) - response := httptest.NewRecorder() - writeDeploymentError(response, request, deployment.ErrLocalNodeConfigured) - if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "runtime_local_node_configured") || !strings.Contains(response.Body.String(), "previous release") { - t.Fatal(response.Code, response.Body.String()) - } -} - // nodeCapacity rejects max_active outside the deployment's node capacity // bounds. func nodeCapacity(active int) error { diff --git a/services/core/internal/api/saved_configuration.go b/services/core/internal/api/saved_configuration.go index 37463d1bc..167e04753 100644 --- a/services/core/internal/api/saved_configuration.go +++ b/services/core/internal/api/saved_configuration.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "slices" "unicode/utf8" v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" @@ -47,19 +46,10 @@ func resolveSavedFields(input v1.CreateAgentRequest) (agents.CreateCommand, erro return agents.CreateCommand{}, err } if input.ServiceTier != nil { - if !slices.Contains([]string{"auto", "default", "flex", "priority", "fast"}, *input.ServiceTier) { - return agents.CreateCommand{}, errors.New("service_tier must be auto, default, flex, priority or fast.") - } cfg.ServiceTier = *input.ServiceTier } if input.Reasoning != nil { cfg.Reasoning = *input.Reasoning - if cfg.Reasoning.Effort != nil && !slices.Contains([]string{"none", "minimal", "low", "medium", "high", "xhigh", "max"}, *cfg.Reasoning.Effort) { - return agents.CreateCommand{}, errors.New("reasoning.effort is not a supported protocol value.") - } - if cfg.Reasoning.Summary != nil && !slices.Contains([]string{"concise", "detailed", "auto"}, *cfg.Reasoning.Summary) { - return agents.CreateCommand{}, errors.New("reasoning.summary must be concise, detailed or auto.") - } } // Model-derived effort resolution is a recorded gap. Do not manufacture a // default from the operator's execution engine or another model's catalog. @@ -79,25 +69,23 @@ func resolveSavedFields(input v1.CreateAgentRequest) (agents.CreateCommand, erro return result, err } +// resolveSavedMultiAgent and resolveText read values whose pinned shape was +// checked at decode; only the uint32 bound of max_concurrent_subagents remains. func resolveSavedMultiAgent(raw json.RawMessage) (v1.MultiAgentConfig, error) { result := v1.MultiAgentConfig{} if len(raw) == 0 || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { return result, nil } - var input struct { - Enabled *bool `json:"enabled"` - Max json.RawMessage `json:"max_concurrent_subagents"` - } - if decodeInputObject(raw, &input, "enabled", "max_concurrent_subagents") != nil || input.Enabled == nil { - return result, errors.New("multi_agent requires enabled as a boolean.") - } - maximum := uint32(6) - if len(input.Max) > 0 && (bytes.Equal(bytes.TrimSpace(input.Max), []byte("null")) || json.Unmarshal(input.Max, &maximum) != nil || maximum == 0) { + input := struct { + Enabled bool `json:"enabled"` + Max uint32 `json:"max_concurrent_subagents"` + }{Max: 6} + if json.Unmarshal(raw, &input) != nil { return result, errors.New("max_concurrent_subagents must be an integer from 1 to 4294967295.") } - result.Enabled = *input.Enabled + result.Enabled = input.Enabled if result.Enabled { - value := int(maximum) + value := int(input.Max) result.MaxConcurrentSubagents = &value } return result, nil @@ -109,37 +97,13 @@ func resolveText(input *v1.TextConfigInput) (v1.TextConfig, error) { return result, nil } if input.Verbosity != nil { - if err := validateTextVerbosity(*input.Verbosity); err != nil { - return result, err - } result.Verbosity = *input.Verbosity } if len(input.Format) == 0 || bytes.Equal(bytes.TrimSpace(input.Format), []byte("null")) { return result, nil } - result.Format = v1.TextFormat{} - if decodeInputObject(input.Format, &result.Format, "type", "schema") != nil { - return result, errors.New("text.format must be a supported format object.") - } - switch result.Format.Type { - case "text": - if len(result.Format.Schema) > 0 { - return result, errors.New("text format does not accept schema.") - } - case "json_schema": - var schema map[string]json.RawMessage - if json.Unmarshal(result.Format.Schema, &schema) != nil || schema == nil { - return result, errors.New("json_schema format requires a schema object.") - } - default: - return result, errors.New("text.format.type must be text or json_schema.") + if err := json.Unmarshal(input.Format, &result.Format); err != nil { + return result, &storedDataError{err} } return result, nil } - -func validateTextVerbosity(value string) error { - if !slices.Contains([]string{"low", "medium", "high"}, value) { - return errors.New("text.verbosity must be low, medium or high.") - } - return nil -} diff --git a/services/core/internal/api/saved_tools.go b/services/core/internal/api/saved_tools.go index 707e23253..e15a41c7e 100644 --- a/services/core/internal/api/saved_tools.go +++ b/services/core/internal/api/saved_tools.go @@ -7,34 +7,34 @@ import ( v1 "github.com/MiniMax-AI/OpenAgentCore/contracts/agents-api/v1" ) +// resolveSavedTools canonicalizes tools whose pinned shape was checked at the +// /v1 boundary. The default case guards a pinned tool type that Core does not +// resolve yet. func resolveSavedTools(input []json.RawMessage) ([]json.RawMessage, error) { tools := make([]json.RawMessage, 0, len(input)) for _, raw := range input { var kind struct { Type string `json:"type"` } - if json.Unmarshal(raw, &kind) != nil { - return nil, errors.New("tools must contain tool objects.") + if err := json.Unmarshal(raw, &kind); err != nil { + return nil, &storedDataError{err} } var value json.RawMessage switch kind.Type { case "function": var function v1.FunctionToolInput - if decodeInputObject(raw, &function, "type", "name", "description", "parameters", "defer_loading") != nil { - return nil, errors.New("Invalid function tool fields.") + if err := json.Unmarshal(raw, &function); err != nil { + return nil, &storedDataError{err} } - resolved, _, err := resolveFunction(function) + resolved, err := resolveFunction(function) if err != nil { return nil, err } value = resolved case "tool_search": - if decodeInputObject(raw, &kind, "type") != nil { - return nil, errors.New("tool_search only accepts type.") - } value, _ = json.Marshal(kind) case "programmatic_tool_calling": - resolved, err := resolveProgrammaticTool(raw) + resolved, _, err := resolveProgrammaticTool(raw) if err != nil { return nil, err } diff --git a/services/core/internal/api/session_agent.go b/services/core/internal/api/session_agent.go index bd95dfa65..279a25894 100644 --- a/services/core/internal/api/session_agent.go +++ b/services/core/internal/api/session_agent.go @@ -36,7 +36,7 @@ func resolveSessionAgent(input sessionRequest, saved *v1.SavedAgent) (v1.Agent, } var override v1.SavedAgentConfiguration if err := json.Unmarshal(resolved.Configuration, &override); err != nil { - return v1.Agent{}, err + return v1.Agent{}, &storedDataError{err} } cfg := override if saved != nil { @@ -84,9 +84,6 @@ func admitSessionAgent(cfg v1.SavedAgentConfiguration) (v1.Agent, error) { if cfg.ServiceTier != "auto" { return v1.Agent{}, errors.New("Execution currently supports service_tier=auto only.") } - if err := validateTextVerbosity(cfg.Text.Verbosity); err != nil { - return v1.Agent{}, err - } tools, err := resolveSessionTools(cfg.Tools) if err != nil { return v1.Agent{}, err diff --git a/services/core/internal/api/session_model_configuration.go b/services/core/internal/api/session_model_configuration.go index 4cde53a67..5134a4528 100644 --- a/services/core/internal/api/session_model_configuration.go +++ b/services/core/internal/api/session_model_configuration.go @@ -41,7 +41,7 @@ func (h *Handler) prepareSessionModelConfiguration(ctx context.Context, input *s if errors.As(err, &configurationError) { return configurationError } - return &modelProviderDefaultsError{err} + return &storedDataError{err} } } input.modelSource = "session" diff --git a/services/core/internal/api/session_model_defaults.go b/services/core/internal/api/session_model_defaults.go index 93bdffd7d..20409c8e1 100644 --- a/services/core/internal/api/session_model_defaults.go +++ b/services/core/internal/api/session_model_defaults.go @@ -30,7 +30,7 @@ func (h *Handler) sessionAgentDefaults(ctx context.Context, tenant string, input } saved := &v1.SavedAgent{ID: resource.ID} if err := json.Unmarshal(resource.Configuration, &saved.SavedAgentConfiguration); err != nil { - return nil, nil, err + return nil, nil, &storedDataError{err} } if inherit && saved.XAgentsCore != nil && saved.XAgentsCore.ModelProvider != nil && provider == nil { return nil, nil, errors.New("saved agent model provider bundle is missing") @@ -44,13 +44,6 @@ type modelProviderRequiredError struct{ message string } func (e *modelProviderRequiredError) Error() string { return e.message } -// modelProviderDefaultsError carries a storage or decryption failure while -// reading the deployment default; it is reported as a service error. -type modelProviderDefaultsError struct{ err error } - -func (e *modelProviderDefaultsError) Error() string { return e.err.Error() } -func (e *modelProviderDefaultsError) Unwrap() error { return e.err } - func modelProviderRequired(environment, engine string) error { if environment == "self_hosted" { return &modelProviderRequiredError{"self_hosted Sessions need a model provider for harness " + engine + ": pass x_agents_core.model_provider or use an Agent that has one saved. Deployment default model providers apply to openai_hosted and none Sessions, never to self_hosted."} diff --git a/services/core/internal/api/session_request.go b/services/core/internal/api/session_request.go index d84edd4dc..d8971677d 100644 --- a/services/core/internal/api/session_request.go +++ b/services/core/internal/api/session_request.go @@ -93,9 +93,6 @@ func (request decodedSessionRequest) validated() (sessionRequest, error) { if err := json.Unmarshal(request.Agent, &input.agentFields); err != nil { return input, sessions.ErrInvalidInput } - if _, supplied := input.agentFields["model"]; supplied && input.Agent.Model == nil { - return input, sessions.ErrInvalidInput - } } if len(request.AgentID) > 0 { var id string diff --git a/services/core/internal/api/session_tools.go b/services/core/internal/api/session_tools.go index 16e9f0b49..c66888e99 100644 --- a/services/core/internal/api/session_tools.go +++ b/services/core/internal/api/session_tools.go @@ -18,8 +18,8 @@ func resolveSessionTools(input []json.RawMessage) ([]json.RawMessage, error) { var kind struct { Type string `json:"type"` } - if json.Unmarshal(raw, &kind) != nil { - return nil, errors.New("Invalid execution tool configuration.") + if err := json.Unmarshal(raw, &kind); err != nil { + return nil, &storedDataError{err} } switch kind.Type { case "programmatic_tool_calling", "web_search": @@ -32,15 +32,10 @@ func resolveSessionTools(input []json.RawMessage) ([]json.RawMessage, error) { if kind.Type == "web_search" { resolved, err = resolveDisabledWebSearch(raw) } else { - resolved, err = resolveProgrammaticTool(raw) - var value struct { - Enabled bool `json:"enabled"` - } - if err == nil { - _ = json.Unmarshal(resolved, &value) - if value.Enabled { - err = errors.New("Programmatic tool calling is not qualified for execution.") - } + var enabled bool + resolved, enabled, err = resolveProgrammaticTool(raw) + if err == nil && enabled { + err = errors.New("Programmatic tool calling is not qualified for execution.") } } if err != nil { @@ -48,8 +43,8 @@ func resolveSessionTools(input []json.RawMessage) ([]json.RawMessage, error) { } tools[i] = resolved case "tool_search": - if search || decodeInputObject(raw, &kind, "type") != nil { - return nil, errors.New("Execution requires one type-only tool_search declaration.") + if search { + return nil, errors.New("Execution requires one tool_search declaration.") } search = true tools[i], _ = json.Marshal(kind) @@ -59,15 +54,18 @@ func resolveSessionTools(input []json.RawMessage) ([]json.RawMessage, error) { return nil, err } var tool v1.MCPTool - if json.Unmarshal(resolved, &tool) != nil || servers[tool.ServerLabel] { + if err := json.Unmarshal(resolved, &tool); err != nil { + return nil, &storedDataError{err} + } + if servers[tool.ServerLabel] { return nil, errors.New("Execution requires distinct MCP server labels.") } servers[tool.ServerLabel] = true tools[i] = resolved case "function": var function v1.FunctionToolInput - if decodeInputObject(raw, &function, "type", "name", "description", "parameters", "defer_loading") != nil { - return nil, errors.New("Invalid execution function fields.") + if err := json.Unmarshal(raw, &function); err != nil { + return nil, &storedDataError{err} } functions = append(functions, function) positions = append(positions, i) diff --git a/services/core/internal/api/validation_errors_test.go b/services/core/internal/api/validation_errors_test.go index fc9b45aac..e0ec582f7 100644 --- a/services/core/internal/api/validation_errors_test.go +++ b/services/core/internal/api/validation_errors_test.go @@ -167,10 +167,15 @@ func TestMetadataValidationUsesOfficialFields(t *testing.T) { if want := map[bool]int{true: 0, false: 3}[op.limited]; s.writes != want { t.Fatalf("rejected metadata reached storage: %d writes", s.writes) } - // Type errors precede the generic whole-body error. + // Type errors precede the generic whole-body error. Agent bodies follow + // the pinned shape walk, which reports unknown members first. body := strings.Replace(fmt.Sprintf(op.body, `{"k":1}`), "{", `{"unsupported_field":true,`, 1) + want := "metadata.k" + if strings.HasPrefix(op.name, "agent") { + want = "unsupported_field" + } w := credentialRequest(h, op.method, op.path, body) - if code, param, _ := errorFields(t, w); w.Code != http.StatusBadRequest || code != "invalid_request_error" || param == nil || *param != "metadata.k" { + if code, param, _ := errorFields(t, w); w.Code != http.StatusBadRequest || code != "invalid_request_error" || param == nil || *param != want { t.Fatalf("metadata type did not precede generic error: %d %s", w.Code, w.Body) } for _, tc := range accepted { diff --git a/services/core/internal/db/queries/runtime_deployment.sql b/services/core/internal/db/queries/runtime_deployment.sql index 36f23dc1c..382d55726 100644 --- a/services/core/internal/db/queries/runtime_deployment.sql +++ b/services/core/internal/db/queries/runtime_deployment.sql @@ -1,10 +1,6 @@ -- name: LockRuntimeDeployment :one SELECT * FROM runtime_deployment WHERE singleton = true FOR UPDATE; --- name: SetRuntimeDeployment :exec -UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true; - -- name: CountRuntimeDeploymentResources :one SELECT (SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index 403741dce..334617404 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -20,7 +20,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid AND p.released_at IS NULL - AND NOT (SELECT admission_paused FROM runtime_deployment) + AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 604a338e0..b6c45b2ce 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -1,6 +1,3 @@ --- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true; - -- name: GetRuntimeDeployment :one SELECT * FROM runtime_deployment WHERE singleton=true; diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d4b6a6e21..6a1ccbeef 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -39,14 +39,6 @@ SELECT clock_timestamp()::timestamptz AS observed_at, FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1; --- name: CountRuntimeComputeReservations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; - --- name: CountRuntimeRetainedAllocations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released'; - -- name: RuntimeComputeBlocksAdmission :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id diff --git a/services/core/internal/db/queries/sandbox_deployment_setup.sql b/services/core/internal/db/queries/sandbox_deployment_setup.sql index 142f0720f..09bd91540 100644 --- a/services/core/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/core/internal/db/queries/sandbox_deployment_setup.sql @@ -1,5 +1,5 @@ -- name: ClaimWebSandboxDeployment :exec -UPDATE runtime_deployment SET installation_id=$1, web_managed=true, +UPDATE runtime_deployment SET installation_id=$1, owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true; -- name: InitializeSandboxDeployment :exec diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 3fad61493..0c2938bb1 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -1,6 +1,6 @@ -- name: StartSandboxReset :exec WITH clock AS MATERIALIZED (SELECT clock_timestamp() AS at) -UPDATE runtime_deployment SET admission_paused = true, reset_clear = sqlc.arg(clear), +UPDATE runtime_deployment SET reset_clear = sqlc.arg(clear), reset_requested_at = clock.at, reset_deadline_at = CASE WHEN sqlc.arg(clear)::text = 'auto' THEN clock.at + make_interval(secs => sqlc.arg(deadline_seconds)::int) END, @@ -14,7 +14,7 @@ UPDATE runtime_deployment SET reset_clear = 'force', reset_forced_at = clock.at, FROM clock WHERE singleton = true AND reset_clear = 'auto'; -- name: CancelSandboxReset :exec -UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, +UPDATE runtime_deployment SET reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true; @@ -24,7 +24,7 @@ UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode specification = '{}', idle_seconds = 0, retention_seconds = 0, provider_config = '{}'::jsonb, provider_metadata = '{}'::jsonb, provider_credential = NULL, generation = generation + 1, owner_epoch = owner_epoch + 1, - admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true; diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 134170554..048878096 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -239,12 +239,9 @@ type RuntimeDeployment struct { Singleton bool `json:"singleton"` InstallationID pgtype.UUID `json:"installation_id"` BackendFingerprint string `json:"backend_fingerprint"` - AdmissionPaused bool `json:"admission_paused"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` ProviderKind string `json:"provider_kind"` - LocalNodeID pgtype.UUID `json:"local_node_id"` OwnerEpoch int64 `json:"owner_epoch"` - WebManaged bool `json:"web_managed"` IdleSeconds int64 `json:"idle_seconds"` RetentionSeconds int64 `json:"retention_seconds"` Generation int64 `json:"generation"` diff --git a/services/core/internal/db/sqlc/runtime_deployment.sql.go b/services/core/internal/db/sqlc/runtime_deployment.sql.go index e8f076e8b..15d71eef4 100644 --- a/services/core/internal/db/sqlc/runtime_deployment.sql.go +++ b/services/core/internal/db/sqlc/runtime_deployment.sql.go @@ -7,8 +7,6 @@ package sqlc import ( "context" - - "github.com/jackc/pgx/v5/pgtype" ) const countAddressBindings = `-- name: CountAddressBindings :one @@ -55,7 +53,7 @@ func (q *Queries) CountRuntimeDeploymentResources(ctx context.Context) (CountRun } const lockRuntimeDeployment = `-- name: LockRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true FOR UPDATE +SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true FOR UPDATE ` func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -65,12 +63,9 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, - &i.LocalNodeID, &i.OwnerEpoch, - &i.WebManaged, &i.IdleSeconds, &i.RetentionSeconds, &i.Generation, @@ -87,19 +82,3 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, ) return i, err } - -const setRuntimeDeployment = `-- name: SetRuntimeDeployment :exec -UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true -` - -type SetRuntimeDeploymentParams struct { - InstallationID pgtype.UUID `json:"installation_id"` - BackendFingerprint string `json:"backend_fingerprint"` - AdmissionPaused bool `json:"admission_paused"` -} - -func (q *Queries) SetRuntimeDeployment(ctx context.Context, arg SetRuntimeDeploymentParams) error { - _, err := q.db.Exec(ctx, setRuntimeDeployment, arg.InstallationID, arg.BackendFingerprint, arg.AdmissionPaused) - return err -} diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index 6036aa233..e4bc7dae0 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -150,7 +150,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid AND p.released_at IS NULL - AND NOT (SELECT admission_paused FROM runtime_deployment) + AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index dc5d98088..e8ee97976 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -119,7 +119,7 @@ func (q *Queries) DisconnectRuntimeNode(ctx context.Context, arg DisconnectRunti } const getRuntimeDeployment = `-- name: GetRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton=true +SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton=true ` func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -129,12 +129,9 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, - &i.LocalNodeID, &i.OwnerEpoch, - &i.WebManaged, &i.IdleSeconds, &i.RetentionSeconds, &i.Generation, @@ -526,20 +523,6 @@ func (q *Queries) RemoveRuntimeNode(ctx context.Context, id pgtype.UUID) error { return err } -const setRuntimeManagerDeployment = `-- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true -` - -type SetRuntimeManagerDeploymentParams struct { - ProviderKind string `json:"provider_kind"` - LocalNodeID pgtype.UUID `json:"local_node_id"` -} - -func (q *Queries) SetRuntimeManagerDeployment(ctx context.Context, arg SetRuntimeManagerDeploymentParams) error { - _, err := q.db.Exec(ctx, setRuntimeManagerDeployment, arg.ProviderKind, arg.LocalNodeID) - return err -} - const setRuntimeObservation = `-- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released' ` diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 5c4867839..ed6b5bb50 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -27,30 +27,6 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara return err } -const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' -` - -func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { - row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) - var count int64 - err := row.Scan(&count) - return count, err -} - -const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' -` - -func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { - row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) - var count int64 - err := row.Scan(&count) - return count, err -} - const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, diff --git a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go index 48cb02cc5..de2ee8d6f 100644 --- a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -21,7 +21,7 @@ func (q *Queries) AdvanceSandboxOwnerEpoch(ctx context.Context) error { } const claimWebSandboxDeployment = `-- name: ClaimWebSandboxDeployment :exec -UPDATE runtime_deployment SET installation_id=$1, web_managed=true, +UPDATE runtime_deployment SET installation_id=$1, owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true ` diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 49512c3f6..3791f0e5d 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -12,7 +12,7 @@ import ( ) const cancelSandboxReset = `-- name: CancelSandboxReset :exec -UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, +UPDATE runtime_deployment SET reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true @@ -28,7 +28,7 @@ UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode specification = '{}', idle_seconds = 0, retention_seconds = 0, provider_config = '{}'::jsonb, provider_metadata = '{}'::jsonb, provider_credential = NULL, generation = generation + 1, owner_epoch = owner_epoch + 1, - admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true @@ -51,7 +51,7 @@ func (q *Queries) ForceSandboxReset(ctx context.Context) error { } const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one -WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true LIMIT 1), +WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true LIMIT 1), observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, @@ -89,7 +89,7 @@ held AS ( ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name ) -SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused, d.updated_at, d.provider_kind, d.local_node_id, d.owner_epoch, d.web_managed, d.idle_seconds, d.retention_seconds, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, +SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.updated_at, d.provider_kind, d.owner_epoch, d.idle_seconds, d.retention_seconds, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, jsonb_build_object( @@ -128,12 +128,9 @@ func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxD &i.RuntimeDeployment.Singleton, &i.RuntimeDeployment.InstallationID, &i.RuntimeDeployment.BackendFingerprint, - &i.RuntimeDeployment.AdmissionPaused, &i.RuntimeDeployment.UpdatedAt, &i.RuntimeDeployment.ProviderKind, - &i.RuntimeDeployment.LocalNodeID, &i.RuntimeDeployment.OwnerEpoch, - &i.RuntimeDeployment.WebManaged, &i.RuntimeDeployment.IdleSeconds, &i.RuntimeDeployment.RetentionSeconds, &i.RuntimeDeployment.Generation, @@ -230,7 +227,7 @@ func (q *Queries) SessionBlocksAutoReset(ctx context.Context, sessionID pgtype.U const startSandboxReset = `-- name: StartSandboxReset :exec WITH clock AS MATERIALIZED (SELECT clock_timestamp() AS at) -UPDATE runtime_deployment SET admission_paused = true, reset_clear = $1, +UPDATE runtime_deployment SET reset_clear = $1, reset_requested_at = clock.at, reset_deadline_at = CASE WHEN $1::text = 'auto' THEN clock.at + make_interval(secs => $2::int) END, diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index cdb44204d..14be17800 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -322,14 +322,14 @@ func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { findAllocation: func() (Allocation, bool, error) { return Allocation{}, false, nil }, lockDeployment: func() (placement.Deployment, error) { return deployment, nil }} } - paused := fresh() - paused.lockDeployment = func() (placement.Deployment, error) { + resetting := fresh() + resetting.lockDeployment = func() (placement.Deployment, error) { d := deployment - d.AdmissionPaused = true + d.Resetting = true return d, nil } - if _, err := allocationOperations(t, paused, sessions.LockedSession{}, nil).ReserveAllocation(t.Context(), key, installation, testCredentialHash()); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("paused admission reserved an allocation", err) + if _, err := allocationOperations(t, resetting, sessions.LockedSession{}, nil).ReserveAllocation(t.Context(), key, installation, testCredentialHash()); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("a resetting deployment reserved an allocation", err) } released := fresh() released.loadReserved = func() (placement.Reserved, error) { diff --git a/services/core/internal/deployment/errors.go b/services/core/internal/deployment/errors.go index 1875e9716..ff90ef7b4 100644 --- a/services/core/internal/deployment/errors.go +++ b/services/core/internal/deployment/errors.go @@ -20,7 +20,6 @@ var ( ErrNotConfigured = errors.New("the sandbox deployment is not configured") ErrNodeInUse = errors.New("sandbox node retains resources") ErrNodeCredential = errors.New("invalid sandbox node credential") - ErrLocalNodeConfigured = errors.New("local sandbox node is enabled in deployment configuration") // ErrAllocationConflict rejects an allocation change whose owner no longer // matches the stored allocation, device binding, state or compute revision, // or a replay for another installation. diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 1b69cee68..0c1ef29fa 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "math" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -41,7 +40,7 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) return err } if d.InstallationID != "" { - if !d.WebManaged || d.InstallationID != id { + if d.InstallationID != id { return ErrConflict } } else { @@ -62,125 +61,19 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) }) } -// ConfigureProcess records the deployment process configuration selects, -// before the Worker starts. AdmissionPaused must be committed for the old -// installation before any switch. A nil selection never forgets the previous -// identity or unresolved resources. -func (e *ExecutionOperations) ConfigureProcess(ctx context.Context, selected *ProcessDeployment) error { - var installation string - if selected != nil { - copy := *selected - selected = © - id, err := parseID(selected.InstallationID) - if err != nil { - return err - } - installation = id - if !validDigest(selected.BackendFingerprint) { - return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) - } - } +// RequireUnclaimed lets an execution owner without sandbox runtimes start +// only on a deployment no installation has claimed. +func (e *ExecutionOperations) RequireUnclaimed(ctx context.Context) error { return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { - previous, err := tx.LoadDeployment() + d, err := tx.LoadDeployment() if err != nil { return err } - if previous.WebManaged { + if d.InstallationID != "" { return ErrConflict } - if selected != nil && previous.InstallationID == installation && previous.BackendFingerprint == selected.BackendFingerprint && (previous.Provider == "" || selected.ProviderKind == previous.Provider) { - if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { - return err - } - return e.configureManager(tx, previous, selected, installation) - } - resources, err := tx.CountResources() - if err != nil { - return err - } - if selected == nil { - if previous.InstallationID != "" && (resources.Allocations != 0 || resources.Pending != 0) { - return fmt.Errorf("cannot disable managed sandbox provider: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - return nil - } - if previous.InstallationID == "" { - if resources.Allocations != 0 { - return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) - } - } else { - if !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot switch sandbox installation: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - } - if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { - return err - } - return e.configureManager(tx, previous, selected, installation) - }) -} - -// configureManager records the node provider and the local node process -// configuration selects. -func (e *ExecutionOperations) configureManager(tx DeploymentTx, previous Record, selected *ProcessDeployment, installation string) error { - if selected.ProviderKind == "" { return nil - } - isNode, err := e.service.registry.IsNode(selected.ProviderKind) - if err != nil { - return err - } - if !isNode { - return ErrInvalidInput - } - if previous.Provider == "" { - resources, err := tx.CountResources() - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot adopt historical sandbox resources: keep the original Core responsible for retained resources and install this release separately") - } - } - var localNode string - if selected.LocalNodeID != "" { - id, err := parseID(selected.LocalNodeID) - if err != nil { - return err - } - localNode = id - if previous.LocalNodeID != "" && previous.LocalNodeID != id { - resources, err := tx.CountResources() - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 || !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("local sandbox node identity changed: restore its original state directory; replacement requires maintenance and no retained resources") - } - } - if !validDigest(selected.LocalCredentialSHA256) { - return ErrInvalidInput - } - if err := validateNode("Local", selected.LocalMaxActive, selected.LocalMaxRetained); err != nil { - return err - } - n, err := tx.LoadNode(id) - if errors.Is(err, ErrNotFound) { - _, err = tx.InsertNode(NewNode{ID: id, InstallationID: installation, Name: "Local", BackendFingerprint: selected.BackendFingerprint, CredentialDigest: selected.LocalCredentialSHA256, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) - } else if err == nil { - if n.InstallationID != installation || n.BackendFingerprint != selected.BackendFingerprint || n.CredentialDigest != selected.LocalCredentialSHA256 { - return fmt.Errorf("local sandbox node identity does not match the retained backend") - } - err = tx.UpdateNode(id, NodeLimits{Name: n.Name, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) - } - if err != nil { - return err - } - } - return tx.SetManagerDeployment(selected.ProviderKind, localNode) + }) } // CheckSetup rejects a stale or reset deployment before provider preparation. diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 526945814..3ccdb1d13 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -76,29 +76,27 @@ func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(De } type fakeReader struct { - t testing.TB - deployment func(context.Context) (Record, error) - snapshot func(context.Context) (Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) - generations func(context.Context, int64) ([]GenerationRecord, error) - nodes func(context.Context) ([]NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) - readNodes func(context.Context, func(NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]ResetSession, error) - addressBindings func(context.Context, string) (AddressBindings, error) - environmentAllocation func(context.Context, AllocationKey) (Allocation, error) - credentialAllocations func(context.Context, string) ([]Allocation, error) - observationSessions func(context.Context, string, int) (ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) - activity func(context.Context, string) (Activity, error) - countComputeReservations func(context.Context, string) (int64, error) - countRetainedAllocations func(context.Context, string) (int64, error) + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]ResetSession, error) + addressBindings func(context.Context, string) (AddressBindings, error) + environmentAllocation func(context.Context, AllocationKey) (Allocation, error) + credentialAllocations func(context.Context, string) ([]Allocation, error) + observationSessions func(context.Context, string, int) (ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) + activity func(context.Context, string) (Activity, error) } func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { @@ -353,11 +351,6 @@ type fakeDeploymentTx struct { loadSnapshot func() (Snapshot, error) countResources func() (Resources, error) claimInstallation func(string) error - setProcessDeployment func(string, string, bool) error - setManagerDeployment func(string, string) error - loadNode func(string) (StoredNode, error) - insertNode func(NewNode) (StoredNode, error) - updateNode func(string, NodeLimits) error saveSelection func(SelectionRecord) error recordConfigurationMetadata func(json.RawMessage) error retainGeneration func() error @@ -399,41 +392,6 @@ func (f *fakeDeploymentTx) ClaimInstallation(installationID string) error { return f.claimInstallation(installationID) } -func (f *fakeDeploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { - if f.setProcessDeployment == nil { - unexpected(f.t, "SetProcessDeployment") - } - return f.setProcessDeployment(installationID, backendFingerprint, admissionPaused) -} - -func (f *fakeDeploymentTx) SetManagerDeployment(provider, localNodeID string) error { - if f.setManagerDeployment == nil { - unexpected(f.t, "SetManagerDeployment") - } - return f.setManagerDeployment(provider, localNodeID) -} - -func (f *fakeDeploymentTx) LoadNode(id string) (StoredNode, error) { - if f.loadNode == nil { - unexpected(f.t, "LoadNode") - } - return f.loadNode(id) -} - -func (f *fakeDeploymentTx) InsertNode(node NewNode) (StoredNode, error) { - if f.insertNode == nil { - unexpected(f.t, "InsertNode") - } - return f.insertNode(node) -} - -func (f *fakeDeploymentTx) UpdateNode(id string, limits NodeLimits) error { - if f.updateNode == nil { - unexpected(f.t, "UpdateNode") - } - return f.updateNode(id, limits) -} - func (f *fakeDeploymentTx) SaveSelection(selection SelectionRecord) error { if f.saveSelection == nil { unexpected(f.t, "SaveSelection") @@ -623,20 +581,6 @@ func (f *fakeReader) Activity(ctx context.Context, allocationID string) (Activit return f.activity(ctx, allocationID) } -func (f *fakeReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - if f.countComputeReservations == nil { - unexpected(f.t, "CountComputeReservations") - } - return f.countComputeReservations(ctx, installationID) -} - -func (f *fakeReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - if f.countRetainedAllocations == nil { - unexpected(f.t, "CountRetainedAllocations") - } - return f.countRetainedAllocations(ctx, installationID) -} - // fakeSessionReader serves the Session reads the observation resolver makes; // every other read fails the test. type fakeSessionReader struct { diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index 3f2ab3764..91867c15f 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -133,9 +133,6 @@ func (s *Service) RemoveNode(ctx context.Context, id string) error { if n.Retained != 0 || n.CleanupPending != 0 { return ErrNodeInUse } - if d.LocalNodeID == nodeID { - return ErrLocalNodeConfigured - } return tx.RemoveNode(nodeID) } return ErrNotFound @@ -165,7 +162,7 @@ func (s *Service) CreateEnrollment(ctx context.Context, capacity Capacity) (Enro if d.Reset != nil { return ErrResetInProgress } - if d.Mode != "nodes" || d.AdmissionPaused { + if d.Mode != "nodes" { return ErrConflict } if _, err := s.specification(d); err != nil { @@ -214,7 +211,7 @@ func (s *Service) Enroll(ctx context.Context, token string, input Enrollment) (N if d.Reset != nil { return ErrResetInProgress } - if d.Mode != "nodes" || d.AdmissionPaused || input.Provider != d.Provider { + if d.Mode != "nodes" || input.Provider != d.Provider { return ErrInvalidInput } spec, err := s.specification(d) @@ -426,9 +423,6 @@ func (s *Service) NodeConfiguration(ctx context.Context, nodeID, token string, g if err != nil { return err } - if node == nil && d.AdmissionPaused { - return ErrConflict - } limit, err := s.registry.RetainedLimit(d.Provider, active, retained) if err != nil { return err diff --git a/services/core/internal/deployment/placement/placement.go b/services/core/internal/deployment/placement/placement.go index 433e6f6b8..9b0c4277a 100644 --- a/services/core/internal/deployment/placement/placement.go +++ b/services/core/internal/deployment/placement/placement.go @@ -20,8 +20,7 @@ var ( // admission. ErrResetAdmission = errors.New("hosted admission is paused for a sandbox reset") // ErrAdmissionClosed rejects new hosted work that the deployment cannot - // admit: paused for maintenance, without a valid specification, or for - // another installation. + // admit: without a valid specification, or for another installation. ErrAdmissionClosed = errors.New("environment is no longer available") // ErrPublicURLUnreachable rejects selection and admission when the provider // requires a reachable public origin and the installation is loopback. @@ -67,12 +66,10 @@ func (r *Rules) PublicURL() string { return r.publicURL } // Deployment is the deployment as placement reads it, loaded under the // deployment lock. type Deployment struct { - // InstallationID is empty until an installation is claimed or configured. - InstallationID string - Provider, Mode string - Generation uint64 - WebManaged bool - AdmissionPaused bool + // InstallationID is empty until Web setup claims an installation. + InstallationID string + Provider, Mode string + Generation uint64 // Resetting reports a sandbox reset in progress. Resetting bool Specification json.RawMessage @@ -139,9 +136,6 @@ func (r *Rules) CheckAdmission(d Deployment, installation string) error { if d.Resetting { return ErrResetAdmission } - if d.AdmissionPaused { - return fmt.Errorf("%w: sandbox creation is paused for provider maintenance", ErrAdmissionClosed) - } if d.Provider != "" { var spec sandbox.DeploymentSpec if json.Unmarshal(d.Specification, &spec) != nil || r.declarations.ValidateSpecification(d.Provider, spec) != nil { @@ -157,8 +151,8 @@ func (r *Rules) CheckAdmission(d Deployment, installation string) error { // DecidePlacement chooses the node a new Session's hosted Environment // reserves, or nil when the deployment places no node: in direct mode and // without a provider. It prefers the highest ready generation, then the -// fewest active sandboxes. A Web-managed installation places only on nodes -// enrolled with its public URL; restores still reach the others. +// fewest active sandboxes. It places only on nodes enrolled with the public +// URL; restores still reach the others. func (r *Rules) DecidePlacement(d Deployment, nodes []Node) (*Placement, error) { if d.Resetting { return nil, ErrResetAdmission @@ -171,26 +165,20 @@ func (r *Rules) DecidePlacement(d Deployment, nodes []Node) (*Placement, error) } } if d.Mode == "direct" { - if d.AdmissionPaused { - return nil, ErrNodeUnavailable - } return nil, nil } if d.Provider == "" { - if d.WebManaged { + if d.InstallationID != "" { return nil, ErrNodeUnavailable } return nil, nil } - if d.AdmissionPaused { - return nil, ErrNodeUnavailable - } var chosen *Node preparing := false for i := range nodes { n := &nodes[i] free := n.Active < int64(n.MaxActive) && n.Retained < int64(n.MaxRetained) - reachable := !d.WebManaged || n.CoreURL == r.publicURL + reachable := n.CoreURL == r.publicURL if n.Online && n.TargetState == "preparing" && free && reachable { preparing = true } diff --git a/services/core/internal/deployment/placement/placement_test.go b/services/core/internal/deployment/placement/placement_test.go index ff7212cde..e60242f4d 100644 --- a/services/core/internal/deployment/placement/placement_test.go +++ b/services/core/internal/deployment/placement/placement_test.go @@ -93,11 +93,10 @@ func TestCheckAdmission(t *testing.T) { want error message string }{ - "unclaimed admits everything": {Deployment{Resetting: true, AdmissionPaused: true}, installation, nil, ""}, + "unclaimed admits everything": {Deployment{Resetting: true}, installation, nil, ""}, "admitted": {valid, installation, nil, ""}, "new Session": {valid, "", nil, ""}, - "reset": {with(func(d *Deployment) { d.Resetting, d.AdmissionPaused = true, true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, - "paused": {with(func(d *Deployment) { d.AdmissionPaused = true }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation is paused for provider maintenance"}, + "reset": {with(func(d *Deployment) { d.Resetting = true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, "malformed specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`[`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, "rejected specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`{"resources":{"cpus":3}}`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, "no provider": {with(func(d *Deployment) { d.Provider, d.Specification = "", json.RawMessage(`[`) }), installation, nil, ""}, @@ -117,7 +116,7 @@ func TestDecidePlacement(t *testing.T) { ready := func(id string, g uint64, active int64) Node { return Node{ID: id, Online: true, ServingReady: true, ReadyGeneration: generation(g), Active: active, MaxActive: 4, Retained: active, MaxRetained: 4, CoreURL: publicURL} } - nodes := Deployment{Provider: "docker", Mode: "nodes", WebManaged: true} + nodes := Deployment{InstallationID: "installation", Provider: "docker", Mode: "nodes"} with := func(change func(*Deployment)) Deployment { d := nodes change(&d) @@ -140,16 +139,13 @@ func TestDecidePlacement(t *testing.T) { "reset": {origin(false), publicURL, with(func(d *Deployment) { d.Resetting = true }), nil, nil, ErrResetAdmission}, "loopback public origin": {origin(true), "http://localhost:8091", nodes, []Node{ready("a", 1, 0)}, nil, ErrPublicURLUnreachable}, "direct": {origin(false), publicURL, with(func(d *Deployment) { d.Mode = "direct" }), nil, nil, nil}, - "direct paused": {origin(false), publicURL, with(func(d *Deployment) { d.Mode, d.AdmissionPaused = "direct", true }), nil, nil, ErrNodeUnavailable}, "no provider": {&fakeDeclarations{t: t}, publicURL, Deployment{}, nil, nil, nil}, - "no provider on Web": {&fakeDeclarations{t: t}, publicURL, Deployment{WebManaged: true}, nil, nil, ErrNodeUnavailable}, - "paused": {origin(false), publicURL, with(func(d *Deployment) { d.AdmissionPaused = true }), []Node{ready("a", 1, 0)}, nil, ErrNodeUnavailable}, + "no provider on Web": {&fakeDeclarations{t: t}, publicURL, Deployment{InstallationID: "installation"}, nil, nil, ErrNodeUnavailable}, "no nodes": {origin(false), publicURL, nodes, nil, nil, ErrNodeUnavailable}, "only ineligible nodes": {origin(false), publicURL, nodes, []Node{offline, unready, full, retainedFull, elsewhere, {ID: "never", Online: true, ServingReady: true, MaxActive: 1, MaxRetained: 1, CoreURL: publicURL}}, nil, ErrNodeUnavailable}, "preparing": {origin(false), publicURL, nodes, []Node{offline, preparing}, nil, ErrNodesPreparing}, "highest generation": {origin(false), publicURL, nodes, []Node{ready("old", 1, 0), ready("new", 2, 3), preparing}, &Placement{NodeID: "new", Generation: 2}, nil}, "fewest active": {origin(false), publicURL, nodes, []Node{ready("busy", 2, 3), ready("idle", 2, 1)}, &Placement{NodeID: "idle", Generation: 2}, nil}, - "other address off Web": {origin(false), publicURL, with(func(d *Deployment) { d.WebManaged = false }), []Node{elsewhere}, &Placement{NodeID: "elsewhere", Generation: 9}, nil}, "public origin on public URL": {origin(true), publicURL, nodes, []Node{ready("a", 1, 0)}, &Placement{NodeID: "a", Generation: 1}, nil}, } { got, err := rules(t, test.declarations, test.url).DecidePlacement(test.d, test.nodes) diff --git a/services/core/internal/deployment/rules.go b/services/core/internal/deployment/rules.go index 91717deb0..d4ab12b35 100644 --- a/services/core/internal/deployment/rules.go +++ b/services/core/internal/deployment/rules.go @@ -70,7 +70,7 @@ func checkGeneration(d Record, installation string, generation uint64) error { if d.Generation != generation { return &GenerationStaleError{CurrentGeneration: d.Generation} } - if !d.WebManaged || d.InstallationID != installation { + if d.InstallationID == "" || d.InstallationID != installation { return ErrConflict } return nil diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go index ea2926e65..1c4819293 100644 --- a/services/core/internal/deployment/rules_test.go +++ b/services/core/internal/deployment/rules_test.go @@ -121,7 +121,7 @@ func TestValidateNode(t *testing.T) { func TestCheckGeneration(t *testing.T) { installation := uuid.NewString() - current := Record{InstallationID: installation, WebManaged: true, Generation: 3} + current := Record{InstallationID: installation, Generation: 3} if err := checkGeneration(current, installation, 3); err != nil { t.Fatal(err) } @@ -129,12 +129,12 @@ func TestCheckGeneration(t *testing.T) { if err := checkGeneration(current, installation, 2); !errors.As(err, &stale) || stale.CurrentGeneration != 3 || !errors.Is(err, ErrConflict) { t.Fatalf("stale generation: %v", err) } - process := current - process.WebManaged = false + unclaimed := current + unclaimed.InstallationID = "" for _, c := range []struct { d Record installation string - }{{process, installation}, {current, uuid.NewString()}} { + }{{unclaimed, installation}, {current, uuid.NewString()}} { if err := checkGeneration(c.d, c.installation, 3); !errors.Is(err, ErrConflict) || errors.As(err, &stale) { t.Errorf("checkGeneration(%+v, %s) = %v, want a plain conflict", c.d, c.installation, err) } diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 673a41dac..575f0d79f 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -88,10 +88,10 @@ func (s *Service) Setup(ctx context.Context) (Setup, error) { } func (s *Service) setup(d Record) (Setup, error) { - if !d.WebManaged { + if d.InstallationID == "" { return Setup{}, ErrConflict } - result := Setup{InstallationID: d.InstallationID, Provider: d.Provider, BackendFingerprint: d.BackendFingerprint, Generation: d.Generation, Mode: d.Mode, AdmissionPaused: d.AdmissionPaused} + result := Setup{InstallationID: d.InstallationID, Provider: d.Provider, BackendFingerprint: d.BackendFingerprint, Generation: d.Generation, Mode: d.Mode} if err := json.Unmarshal(d.Specification, &result.Specification); err != nil { return Setup{}, err } diff --git a/services/core/internal/deployment/service_test.go b/services/core/internal/deployment/service_test.go index 9fc5964aa..e3554720f 100644 --- a/services/core/internal/deployment/service_test.go +++ b/services/core/internal/deployment/service_test.go @@ -71,7 +71,7 @@ func webDeployment(t *testing.T, installation, provider string, generation uint6 if err != nil { t.Fatal(err) } - return Record{InstallationID: installation, WebManaged: true, Provider: provider, Generation: generation, Mode: "nodes", Specification: specification, + return Record{InstallationID: installation, Provider: provider, Generation: generation, Mode: "nodes", Specification: specification, Configuration: sandbox.ConfigurationRecord{Public: json.RawMessage(`{}`), Metadata: json.RawMessage(`{}`)}} } @@ -178,15 +178,6 @@ func TestRegistryLookupFailuresPropagate(t *testing.T) { if _, err := service.ListNodes(t.Context()); !errors.Is(err, providers.ErrUnknownProvider) { t.Errorf("ListNodes = %v", err) } - tx := &fakeDeploymentTx{t: t, - loadDeployment: func() (Record, error) { return Record{}, nil }, - countResources: func() (Resources, error) { return Resources{}, nil }, - setProcessDeployment: func(string, string, bool) error { return nil }, - } - process := &ProcessDeployment{ProviderKind: "retired", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64)} - if err := operations(t, testPublicURL, tx).ConfigureProcess(t.Context(), process); !errors.Is(err, providers.ErrUnknownProvider) { - t.Errorf("ConfigureProcess = %v", err) - } } // Setup carries the mode and operations the provider declares. A stored @@ -280,8 +271,6 @@ func TestEnrollChecksTheTokenBeforeTheDeployment(t *testing.T) { receipt := EnrollmentRecord{ID: uuid.NewString(), InstallationID: installation, ExpiresAt: time.Now().Add(10 * time.Minute), MaxActive: 1, MaxRetained: 1} resetting := current resetting.Reset = &ResetState{Clear: "sandboxes", RequestedAt: time.Now()} - paused := current - paused.AdmissionPaused = true unselected := current unselected.Provider = "" consumed, expired, foreign := receipt, receipt, receipt @@ -303,12 +292,11 @@ func TestEnrollChecksTheTokenBeforeTheDeployment(t *testing.T) { }{ {"unknown token while resetting", resetting, EnrollmentRecord{}, ErrNotFound, valid, ErrNodeCredential}, {"consumed token before setup", Record{}, consumed, nil, valid, ErrNodeCredential}, - {"expired token while paused", paused, expired, nil, valid, ErrNodeCredential}, + {"expired token while resetting", resetting, expired, nil, valid, ErrNodeCredential}, {"token of another installation while resetting", resetting, foreign, nil, valid, ErrNodeCredential}, {"token store failure", current, EnrollmentRecord{}, errors.New("database down"), valid, placement.ErrNodeUnavailable}, {"no provider selected", unselected, receipt, nil, valid, placement.ErrNodeUnavailable}, {"reset in progress", resetting, receipt, nil, valid, ErrResetInProgress}, - {"admission paused", paused, receipt, nil, valid, ErrInvalidInput}, {"stale generation", current, receipt, nil, stale, ErrSpecificationMismatch}, // The token stays unused: the fake allows no insert or consumption. {"another Core address", current, receipt, nil, elsewhere, ErrNodeAddressMismatch}, diff --git a/services/core/internal/deployment/session_archive.go b/services/core/internal/deployment/session_archive.go index d2b2578fd..eb33b012d 100644 --- a/services/core/internal/deployment/session_archive.go +++ b/services/core/internal/deployment/session_archive.go @@ -119,13 +119,13 @@ func (e *ExecutionOperations) archiveSession(ctx context.Context, tenantID, sess } // checkArchiveDeployment rejects an archive against a deployment whose -// generation is not the expected one, that Web does not manage or that has no +// generation is not the expected one, that has no installation or that has no // provider. func checkArchiveDeployment(d Record, expectedGeneration uint64) error { if d.Generation != expectedGeneration { return &GenerationStaleError{CurrentGeneration: d.Generation} } - if !d.WebManaged || d.InstallationID == "" { + if d.InstallationID == "" { return ErrConflict } if d.Provider == "" { diff --git a/services/core/internal/deployment/session_archive_test.go b/services/core/internal/deployment/session_archive_test.go index 3ebbd7cdb..2db8c361c 100644 --- a/services/core/internal/deployment/session_archive_test.go +++ b/services/core/internal/deployment/session_archive_test.go @@ -14,14 +14,13 @@ import ( ) func TestCheckArchiveDeployment(t *testing.T) { - managed := Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 4} + managed := Record{InstallationID: "installation", Provider: "docker", Generation: 4} for name, test := range map[string]struct { change func(*Record) want error }{ "current": {func(*Record) {}, nil}, - "stale generation": {func(d *Record) { d.Generation = 5; d.WebManaged = false }, &GenerationStaleError{CurrentGeneration: 5}}, - "process managed": {func(d *Record) { d.WebManaged = false; d.Provider = "" }, ErrConflict}, + "stale generation": {func(d *Record) { d.Generation = 5; d.InstallationID = "" }, &GenerationStaleError{CurrentGeneration: 5}}, "no installation": {func(d *Record) { d.InstallationID = "" }, ErrConflict}, "no provider": {func(d *Record) { d.Provider = "" }, ErrNotConfigured}, } { @@ -225,7 +224,7 @@ func archiveOperations(t *testing.T, tx *fakeArchiveTx, locked sessions.LockedSe } func TestArchiveSession(t *testing.T) { - managed := Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 1} + managed := Record{InstallationID: "installation", Provider: "docker", Generation: 1} hosted := &sessions.Environment{ID: "environment", Status: "connected", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} expired := &sessions.Environment{ID: "environment", Status: "expired", Configuration: hosted.Configuration} live := &Allocation{ID: "allocation", DeviceID: "device", ProviderKey: "installation", State: "running"} @@ -281,7 +280,7 @@ func TestArchiveSession(t *testing.T) { func TestArchiveResetSession(t *testing.T) { requested := time.Unix(100, 0) resetting := func(clear string) Record { - return Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 1, Reset: &ResetState{Clear: clear, RequestedAt: requested}} + return Record{InstallationID: "installation", Provider: "docker", Generation: 1, Reset: &ResetState{Clear: clear, RequestedAt: requested}} } hosted := &sessions.Environment{ID: "environment", Status: "connected", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} failed := &sessions.Environment{ID: "environment", Status: "failed", Configuration: hosted.Configuration} diff --git a/services/core/internal/deployment/setup.go b/services/core/internal/deployment/setup.go index e40852e6c..2f17624b0 100644 --- a/services/core/internal/deployment/setup.go +++ b/services/core/internal/deployment/setup.go @@ -16,8 +16,7 @@ type Setup struct { Generation uint64 // Mode is where the provider runs: "nodes" for enrolled sandbox nodes and // "direct" for a provider Core calls itself. - Mode string - AdmissionPaused bool + Mode string // Operations is the provider's declared operation support, which the node // transport proxies. Operations providercontract.Operations @@ -35,19 +34,6 @@ func (s Setup) selection() sandbox.Selection { return sandbox.Selection{Provider: s.Provider, DeploymentSpec: s.Specification, Configuration: s.Configuration} } -// ProcessDeployment is a deployment selected by process configuration instead of -// Web setup. Its fingerprint describes the backend namespace, never credentials -// or image contents. -type ProcessDeployment struct { - ProviderKind string - LocalNodeID string - LocalCredentialSHA256 string - LocalMaxActive, LocalMaxRetained int - InstallationID string - BackendFingerprint string - AdmissionPaused bool -} - // configurationJSON reports an absent configuration object as {}. func configurationJSON(raw json.RawMessage) json.RawMessage { if len(raw) == 0 { diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 20cff0768..e3b9dc761 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -249,12 +249,6 @@ type Reader interface { // Activity returns the allocation's activity; a missing allocation is // ErrNotFound. Activity(ctx context.Context, allocationID string) (Activity, error) - // CountComputeReservations counts the installation's allocations that - // reserve active compute. - CountComputeReservations(ctx context.Context, installationID string) (int64, error) - // CountRetainedAllocations counts the installation's allocations that - // retain resources. - CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) } // NodeReads loads node authentication facts. @@ -306,13 +300,6 @@ type DeploymentTx interface { // ClaimInstallation reserves the installation for Web setup and fences the // previous owner epoch's node presence. ClaimInstallation(installationID string) error - SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error - // SetManagerDeployment records the node provider and the local node, which - // is empty when there is none. - SetManagerDeployment(provider, localNodeID string) error - LoadNode(id string) (StoredNode, error) - InsertNode(node NewNode) (StoredNode, error) - UpdateNode(id string, limits NodeLimits) error // SaveSelection stores the next generation. It seals a secret bound to the // installation and generation; without a key it returns // credentialcrypto.ErrUnavailable. @@ -348,16 +335,13 @@ type DeploymentTx interface { // Record is the stored deployment. type Record struct { - // InstallationID is empty until an installation is claimed or configured. + // InstallationID is empty until Web setup claims an installation. InstallationID string - WebManaged bool Provider string BackendFingerprint string Generation uint64 OwnerEpoch uint64 Mode string - AdmissionPaused bool - LocalNodeID string IdleSeconds int64 RetentionSeconds int64 // Specification is the stored specification document. diff --git a/services/core/internal/execution/archive_cancellation_cleanup_test.go b/services/core/internal/execution/archive_cancellation_cleanup_test.go index 95353a660..f6e658f96 100644 --- a/services/core/internal/execution/archive_cancellation_cleanup_test.go +++ b/services/core/internal/execution/archive_cancellation_cleanup_test.go @@ -49,7 +49,7 @@ func (waitingCleanupCheckpoint) ProviderOperations() providercontract.Operations } type waitingCleanupCheckpoint struct { - sandbox.CheckpointProvider + sandbox.SandboxProvider beforeKill func() } diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 94bacf49e..6562e189a 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -52,6 +52,15 @@ func unitDeploymentService(t *testing.T) *deployment.Service { return service } +// unusedPreparation is the preparer of a test that submits no sandbox +// selection; preparing one fails the test. +func unusedPreparation(t *testing.T) RuntimeDeploymentPreparer { + return func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) { + t.Error("the test prepared a sandbox selection it did not submit") + return PreparedRuntimeDeployment{}, errors.New("unexpected sandbox selection preparation") + } +} + // deploymentOperations builds the deployment service on storage and reader and // the execution operations on execution. func deploymentOperations(t *testing.T, storage deployment.Storage, reader deployment.Reader, execution deployment.ExecutionStorage) (*deployment.Service, *deployment.ExecutionOperations) { @@ -132,29 +141,27 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - t *testing.T - deployment func(context.Context) (deployment.Record, error) - snapshot func(context.Context) (deployment.Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) - generations func(context.Context, int64) ([]deployment.GenerationRecord, error) - nodes func(context.Context) ([]deployment.NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) - readNodes func(context.Context, func(deployment.NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) - addressBindings func(context.Context, string) (deployment.AddressBindings, error) - environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) - credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) - observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) - activity func(context.Context, string) (deployment.Activity, error) - countComputeReservations func(context.Context, string) (int64, error) - countRetainedAllocations func(context.Context, string) (int64, error) + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) + addressBindings func(context.Context, string) (deployment.AddressBindings, error) + environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) + credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) + observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) + activity func(context.Context, string) (deployment.Activity, error) } func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { @@ -335,17 +342,3 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri } return r.activity(ctx, allocationID) } - -func (r *strictDeploymentReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - if r.countComputeReservations == nil { - return 0, unexpectedDeploymentCall(r.t, "CountComputeReservations") - } - return r.countComputeReservations(ctx, installationID) -} - -func (r *strictDeploymentReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - if r.countRetainedAllocations == nil { - return 0, unexpectedDeploymentCall(r.t, "CountRetainedAllocations") - } - return r.countRetainedAllocations(ctx, installationID) -} diff --git a/services/core/internal/execution/owner_test.go b/services/core/internal/execution/owner_test.go index 57d676ee9..9da8bdb4b 100644 --- a/services/core/internal/execution/owner_test.go +++ b/services/core/internal/execution/owner_test.go @@ -134,7 +134,7 @@ func TestStartWorkerFailureClosesLeaseOnce(t *testing.T) { lease.inner = owner.Lease id := uuid.NewString() service, reader := unusedSessions(t) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} _, err := StartWorker(canceled, dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if ping := owner.Lease.CheckOwnership(t.Context()); !errors.Is(ping, pgunit.ErrLeaseClosed) { t.Error("failed start kept the database lease", ping) @@ -193,7 +193,7 @@ func TestWorkerRunClosesLeaseAfterDrain(t *testing.T) { id := uuid.NewString() // The Worker's first reconciliation scans the Session work. reader, service := testSessions(t, pool, nil) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} worker, err := StartWorker(t.Context(), dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index c8fc32805..98e951c04 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -10,26 +10,21 @@ import ( func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } func (*lifecycleOnlySandbox) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { @@ -62,17 +57,3 @@ func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, s func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*lifecycleOnlySandbox) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_operation_not_supported"} -} -func (*lifecycleOnlySandbox) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "fixture_operation_not_supported"} -} -func (*lifecycleOnlySandbox) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "fixture_operation_not_supported"} -} - -func (*lifecycleOnlySandbox) ObservationProviderType() string { return "fixture" } -func (p *lifecycleOnlySandbox) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 7d08a1b33..c1aeab3c6 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -8,17 +8,16 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) -// RuntimeSuspensionPolicy applies only to an explicitly qualified single-host -// provider. Fixed guest sizing plus MaxActive bounds reserved CPU and memory. +// RuntimeSuspensionPolicy is the idle suspension policy of a provider that +// suspends sandboxes. type RuntimeSuspensionPolicy struct { IdleTimeout time.Duration Retention time.Duration - MaxActive int - MaxRetained int } type runtimeCompute struct { @@ -30,23 +29,6 @@ type runtimeCompute struct { Rollback bool `json:"rollback,omitempty"` } -func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { - policy := r.config.Suspension - if key != r.config.InstallationID { - return sandbox.ErrOwnership - } - if policy == nil { - return nil - } - count, err := r.reader.CountComputeReservations(ctx, key) - if err != nil { - return err - } - if count >= int64(policy.MaxActive) { - return ErrExecutionUnavailable - } - return nil -} func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.Allocation, phase string, state runtimeCompute, until *time.Time) (deployment.Allocation, error) { raw, err := json.Marshal(state) if err != nil { @@ -63,9 +45,9 @@ func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.All return r.deployment.SetCompute(ctx, owner, phase, raw, until, idleTimeout) } func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.Allocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) - if capabilityErr != nil { - return capabilityErr + p := r.config.Provider + if err := providercontract.Require(p, "Initial"); err != nil { + return err } initial, err := p.Initial(ctx, runtimeReference(owner)) if err != nil { @@ -83,9 +65,9 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.A } func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment.Allocation) error { - p, capabilityErr := sandbox.Checkpoint(r.config.Provider) - if capabilityErr != nil { - return capabilityErr + p := r.config.Provider + if err := providercontract.Require(p, "Initial"); err != nil { + return err } var state runtimeCompute if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { @@ -122,7 +104,7 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. } } -func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) if err != nil { return err @@ -189,7 +171,7 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.Checkpoint return r.captureCompute(ctx, p, suspending, state, false) } -func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { +func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { result, err := p.Suspend(ctx, sandbox.SuspendRequest{Reference: runtimeReference(owner), OperationID: state.SuspendID, Source: state.Current, Snapshot: state.Snapshot, ObserveOnly: observeOnly}) if err != nil { return err @@ -222,7 +204,7 @@ func (r *runtimeLifecycle) captureCompute(ctx context.Context, p sandbox.Checkpo return err } -func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { activity, err := r.reader.Activity(ctx, owner.ID) if err != nil { return err @@ -230,9 +212,6 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che if !activity.Busy && !activity.WakeRequested { return nil } - if err := r.computeCapacityForAllocation(ctx, owner); err != nil { - return err - } if state.Snapshot == nil || state.Target != nil { return sandbox.ErrOwnership } @@ -247,7 +226,7 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che } return r.restoreCompute(ctx, p, next, state, false) } -func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { +func (r *runtimeLifecycle) restoreCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute, observeOnly bool) error { if state.Target == nil || state.Snapshot == nil || state.Rollback { return sandbox.ErrOwnership } @@ -274,11 +253,3 @@ func ignoreComputeAbsent(err error) error { } return err } - -// Node-backed restores reserve capacity atomically in SetCompute. -func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, owner deployment.Allocation) error { - if owner.NodeID != "" { - return nil - } - return r.computeCapacity(ctx, owner.ProviderKey) -} diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index 552ede1ed..ed4223d90 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -12,7 +12,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { if state.Rollback { if _, err := p.ResumeCompute(ctx, runtimeReference(owner), state.Current); err != nil { return err @@ -70,7 +70,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.Checkpoint return r.observeConnection(ctx, next) } -func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.CheckpointProvider, owner deployment.Allocation, state runtimeCompute) error { +func (r *runtimeLifecycle) cleanupCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) error { if err := r.lease.CheckOwnership(ctx); err != nil { return err } diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index d6ca1b27c..5e275b09e 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -23,21 +23,17 @@ import ( // BackendFingerprint identifies its namespace independently of mutable sizing. type RuntimeProvider struct { // PublishUnconfigured updates the shared observation cache after reset commit. - PublishUnconfigured func(uint64) - Generation uint64 - Mode string - loadDeployment func(context.Context) (*RuntimeProvider, error) - prepareDeployment RuntimeDeploymentPreparer - ProviderKind string - LocalNodeID string - LocalCredentialSHA256 string - LocalMaxActive, LocalMaxRetained int - CoreURL string - InstallationID string - BackendFingerprint string - Provider sandbox.SandboxProvider - AdmissionPaused bool - Suspension *RuntimeSuspensionPolicy + PublishUnconfigured func(uint64) + Generation uint64 + Mode string + loadDeployment func(context.Context) (*RuntimeProvider, error) + prepareDeployment RuntimeDeploymentPreparer + ProviderKind string + CoreURL string + InstallationID string + BackendFingerprint string + Provider sandbox.SandboxProvider + Suspension *RuntimeSuspensionPolicy } type runtimeLifecycle struct { @@ -67,21 +63,12 @@ func newRuntimeManager(owner Owner, deployments *deployment.Service, deploymentR if config == nil { return nil, nil } - var copied RuntimeProvider - if config.loadDeployment != nil { - id, err := uuid.Parse(config.InstallationID) - if err != nil || id == uuid.Nil || id.String() != config.InstallationID || registry == nil { - return nil, sandbox.ErrInvalid - } - } else { - var err error - copied, err = validatedRuntimeProvider(config, registry) - if err != nil { - return nil, err - } + id, err := uuid.Parse(config.InstallationID) + if err != nil || id == uuid.Nil || id.String() != config.InstallationID || config.loadDeployment == nil || config.prepareDeployment == nil || registry == nil { + return nil, sandbox.ErrInvalid } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway.Registry) (RuntimeProvider, error) { @@ -98,18 +85,15 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. return RuntimeProvider{}, err } copied := *config - if copied.Mode == "" && copied.ProviderKind != "" { - copied.Mode = "nodes" - } - if copied.Mode != "" && copied.Mode != "nodes" && copied.Mode != "direct" { + if copied.ProviderKind == "" || (copied.Mode != "nodes" && copied.Mode != "direct") { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "" || copied.Suspension != nil) { + if copied.Mode == "direct" && copied.Suspension != nil { return RuntimeProvider{}, sandbox.ErrInvalid } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { + if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -187,25 +171,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return deployment.Allocation{}, sandbox.ErrInvalid } key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} - if _, err := r.reader.EnvironmentAllocation(ctx, key); errors.Is(err, deployment.ErrNotFound) { - if r.config.AdmissionPaused && r.config.Generation == 0 { - return deployment.Allocation{}, ErrExecutionUnavailable - } - if err := r.computeFreshCapacity(ctx, providerKey); err != nil { - return deployment.Allocation{}, err - } - if policy := r.config.Suspension; policy != nil && r.config.ProviderKind == "" { - count, err := r.reader.CountRetainedAllocations(ctx, providerKey) - if err != nil { - return deployment.Allocation{}, err - } - if count >= int64(policy.MaxRetained) { - return deployment.Allocation{}, ErrExecutionUnavailable - } - } - } else if err != nil { - return deployment.Allocation{}, err - } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -435,11 +400,3 @@ func runtimeReference(owner deployment.Allocation) sandbox.Reference { func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } - -// Manager deployments reserve capacity with Session placement before provisioning. -func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { - if r.config.ProviderKind != "" { - return nil - } - return r.computeCapacity(ctx, key) -} diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 24f0ec8cb..d7d1af2b2 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -79,7 +79,7 @@ func (m *runtimeManager) node(id string) (*runtimeNode, error) { m.mu.Unlock() return nil, errRuntimeTransition } - if m.closed || (m.loadDeployment != nil && m.config.Provider == nil) || (id == "") != (m.config.ProviderKind == "" || m.config.Mode == "direct") { + if m.closed || m.config.Provider == nil || (id == "") != (m.config.Mode == "direct") { m.mu.Unlock() return nil, ErrExecutionUnavailable } @@ -241,10 +241,8 @@ func (m *runtimeManager) run(ctx context.Context) error { } m.running = true m.mu.Unlock() - if m.loadDeployment != nil { - if err := m.deployment.CollectGenerations(ctx); err != nil { - return err - } + if err := m.deployment.CollectGenerations(ctx); err != nil { + return err } if err := m.resetStep(ctx); err != nil { return err @@ -263,10 +261,8 @@ func (m *runtimeManager) run(ctx context.Context) error { case err := <-m.failed: return err case <-ticker.C: - if m.loadDeployment != nil { - if err := m.deployment.CollectGenerations(ctx); err != nil { - return err - } + if err := m.deployment.CollectGenerations(ctx); err != nil { + return err } if err := m.resetStep(ctx); err != nil { return err diff --git a/services/core/internal/execution/runtime_manager_test.go b/services/core/internal/execution/runtime_manager_test.go index bce4dc075..a01528e43 100644 --- a/services/core/internal/execution/runtime_manager_test.go +++ b/services/core/internal/execution/runtime_manager_test.go @@ -19,10 +19,11 @@ func (heldLease) CancelOperations(_ context.Context, cancel context.CancelFunc) } func (heldLease) Close(context.Context) error { return nil } +// testRuntimeManager models an already loaded node deployment. func testRuntimeManager(t *testing.T) *runtimeManager { t.Helper() ctx, cancel := context.WithCancel(t.Context()) - m := &runtimeManager{lease: heldLease{}, config: RuntimeProvider{ProviderKind: "docker"}, ctx: ctx, cancel: cancel, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)} + m := &runtimeManager{lease: heldLease{}, config: RuntimeProvider{ProviderKind: "docker", Mode: "nodes", Provider: &drainFixtureProvider{}}, loadDeployment: func(context.Context) (*RuntimeProvider, error) { return nil, nil }, ctx: ctx, cancel: cancel, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)} t.Cleanup(func() { m.stop(); m.drain() }) return m } diff --git a/services/core/internal/execution/runtime_pending.go b/services/core/internal/execution/runtime_pending.go index 254b33877..cee0c94a3 100644 --- a/services/core/internal/execution/runtime_pending.go +++ b/services/core/internal/execution/runtime_pending.go @@ -10,9 +10,6 @@ import ( // provisionPending shares the existing lifecycle owner and serial gate. This // also recovers idle Session creation interrupted after its database commit. func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { - if r.config.AdmissionPaused && r.config.Generation == 0 { - return nil - } rows, err := r.reader.UnallocatedEnvironments(ctx, r.nodeID, r.pendingCursor) if err != nil { return err diff --git a/services/core/internal/execution/runtime_retirement_failure_test.go b/services/core/internal/execution/runtime_retirement_failure_test.go index f5e8e69b0..ed2b63489 100644 --- a/services/core/internal/execution/runtime_retirement_failure_test.go +++ b/services/core/internal/execution/runtime_retirement_failure_test.go @@ -32,11 +32,7 @@ func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { owner, _, _, pool := delayedReadWriter(t, &armed, reading, releaseRead) m := testRuntimeManager(t) m.lease = owner.Lease - m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } m.mutationGate = make(chan struct{}, 1) - // This fixture models an already loaded node deployment; its provider is - // needed only for node admission, not for external sandbox operations. - m.config.Provider = &drainFixtureProvider{} original, err := m.node("retiring") if err != nil { t.Fatal(err) @@ -160,7 +156,8 @@ func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { } } -// No provider method is called by the retirement ownership fixture. +// No provider method is called by the runtime manager fixtures; the provider is +// needed only for node admission. type drainFixtureProvider struct{ sandbox.SandboxProvider } // Done is evaluated only after lockMutation's initial admission check, letting @@ -178,7 +175,6 @@ func (c *mutationWaitContext) Done() <-chan struct{} { func TestFailedManagerRejectsAlreadyWaitingMutation(t *testing.T) { m := testRuntimeManager(t) - m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } m.mutationGate = make(chan struct{}, 1) m.mutationGate <- struct{}{} ctx, cancel := context.WithCancel(t.Context()) diff --git a/services/core/internal/execution/sandbox_deployment_drain_test.go b/services/core/internal/execution/sandbox_deployment_drain_test.go index 3104220af..4ace349be 100644 --- a/services/core/internal/execution/sandbox_deployment_drain_test.go +++ b/services/core/internal/execution/sandbox_deployment_drain_test.go @@ -92,8 +92,8 @@ func testLifecycleCancellationPreservesLease(t *testing.T, mode string) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -230,8 +230,8 @@ func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_deployment_setup.go b/services/core/internal/execution/sandbox_deployment_setup.go index 7bff16c58..165198028 100644 --- a/services/core/internal/execution/sandbox_deployment_setup.go +++ b/services/core/internal/execution/sandbox_deployment_setup.go @@ -22,14 +22,11 @@ type PreparedRuntimeDeployment struct { type RuntimeDeploymentPreparer func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) // NewDeferredRuntimeProvider enables Web setup for one fixed installation. The -// loader returns nil until selection, then the committed immutable generation. -// Replacement is serialized by the deployment mutation gate and drain flow. -func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error), prepare ...RuntimeDeploymentPreparer) *RuntimeProvider { - config := &RuntimeProvider{InstallationID: installationID, loadDeployment: load} - if len(prepare) == 1 { - config.prepareDeployment = prepare[0] - } - return config +// loader returns nil until selection, then the committed immutable generation; +// prepare validates each new selection before it is stored. Replacement is +// serialized by the deployment mutation gate and drain flow. +func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error), prepare RuntimeDeploymentPreparer) *RuntimeProvider { + return &RuntimeProvider{InstallationID: installationID, loadDeployment: load, prepareDeployment: prepare} } func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) { @@ -67,9 +64,6 @@ func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input sandbox. // ensureDeployment serializes the first configuration read without holding the // node map lock across database access. All node workers copy this same snapshot. func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) { - if m.loadDeployment == nil { - return true, nil - } ctx, finish, err := m.enter(parent) if err != nil { return false, err @@ -97,7 +91,7 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) if err != nil || config == nil { return false, err } - if config.InstallationID != m.setupInstallationID || config.LocalNodeID != "" || config.loadDeployment != nil || config.ProviderKind == "" { + if config.InstallationID != m.setupInstallationID || config.loadDeployment != nil { return false, sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) @@ -119,9 +113,6 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) // Preparation is outside the manager mutex and all database transactions. A // rejected candidate cannot retire the current generation or its node lanes. func (m *runtimeManager) prepareCandidate(ctx context.Context, input sandbox.Selection) (PreparedRuntimeDeployment, error) { - if m.prepareDeployment == nil { - return PreparedRuntimeDeployment{}, ErrExecutionUnavailable - } setup, err := m.deploymentService.SetupForSelection(m.setupInstallationID, input) if err != nil { return PreparedRuntimeDeployment{}, err @@ -131,7 +122,7 @@ func (m *runtimeManager) prepareCandidate(ctx context.Context, input sandbox.Sel return PreparedRuntimeDeployment{}, err } config := candidate.Config - if config == nil || config.InstallationID != setup.InstallationID || config.ProviderKind != setup.Provider || config.Mode != setup.Mode || config.CoreURL == "" || config.BackendFingerprint != setup.BackendFingerprint || config.LocalNodeID != "" || config.loadDeployment != nil || config.prepareDeployment != nil { + if config == nil || config.InstallationID != setup.InstallationID || config.ProviderKind != setup.Provider || config.Mode != setup.Mode || config.CoreURL == "" || config.BackendFingerprint != setup.BackendFingerprint || config.loadDeployment != nil || config.prepareDeployment != nil { return PreparedRuntimeDeployment{}, sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) @@ -164,7 +155,7 @@ func (m *runtimeManager) publishDeployment(candidate PreparedRuntimeDeployment, m.mu.Lock() defer m.mu.Unlock() config := *candidate.Config - config.Generation, config.AdmissionPaused = committed.Generation, committed.Reset != nil + config.Generation = committed.Generation if m.switching { m.nodes = make(map[string]*runtimeNode) } diff --git a/services/core/internal/execution/sandbox_deployment_setup_test.go b/services/core/internal/execution/sandbox_deployment_setup_test.go index 35271c554..0337c3199 100644 --- a/services/core/internal/execution/sandbox_deployment_setup_test.go +++ b/services/core/internal/execution/sandbox_deployment_setup_test.go @@ -24,14 +24,14 @@ func TestDeferredSandboxDeploymentLoadsOnceBeforeNodeCreation(t *testing.T) { id := uuid.NewString() var selected atomic.Bool var loads atomic.Int32 - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { loads.Add(1) if !selected.Load() { return nil, nil } return configuration, nil - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -74,7 +74,7 @@ func TestDeferredSandboxDeploymentShutdownCancelsLoad(t *testing.T) { close(entered) <-ctx.Done() return nil, ctx.Err() - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -94,13 +94,13 @@ func TestDeferredSandboxProviderFailureKeepsRecoveryAvailable(t *testing.T) { id := uuid.NewString() available := false loadErr := ErrExecutionUnavailable - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { if !available { return nil, loadErr } return configuration, nil - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -125,7 +125,7 @@ func TestRejectedSandboxCandidatePreservesActiveGeneration(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} rejected := errors.New("candidate provider unavailable") m, err := newRuntimeManager(Owner{Lease: heldLease{}}, unitDeploymentService(t), nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) { @@ -192,20 +192,20 @@ func TestCommittedSandboxCandidatePublishesAfterShutdown(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } if err := m.pauseDeployment(t.Context()); err != nil { t.Fatal(err) } - config := &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + config := &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} var published *RuntimeProvider candidate := PreparedRuntimeDeployment{Config: config, Publish: func(value *RuntimeProvider) { published = value }} m.stop() m.publishDeployment(candidate, deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Reset: &deployment.Reset{}}) m.drain() - if m.config.Generation != 2 || !m.config.AdmissionPaused || published == nil || published.Generation != 2 || !published.AdmissionPaused || m.switching { + if m.config.Generation != 2 || published == nil || published.Generation != 2 || m.switching { t.Fatal("committed candidate was lost during shutdown") } if _, _, err := m.enter(t.Context()); !errors.Is(err, ErrExecutionUnavailable) { diff --git a/services/core/internal/execution/sandbox_deployment_switch.go b/services/core/internal/execution/sandbox_deployment_switch.go index 6e18883cf..0dddd2923 100644 --- a/services/core/internal/execution/sandbox_deployment_switch.go +++ b/services/core/internal/execution/sandbox_deployment_switch.go @@ -9,7 +9,7 @@ import ( ) func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { - if m == nil || m.loadDeployment == nil { + if m == nil { return nil, deployment.ErrConflict } m.mu.Lock() @@ -115,7 +115,7 @@ func (m *runtimeManager) activateDeployment(ctx context.Context, expected deploy m.publishEmptyDeployment(expected.InstallationID, expected.Generation) return nil } - if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil || config.LocalNodeID != "" { + if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil { return sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) diff --git a/services/core/internal/execution/sandbox_deployment_switch_test.go b/services/core/internal/execution/sandbox_deployment_switch_test.go index 2d31d3d32..6f9a1d14d 100644 --- a/services/core/internal/execution/sandbox_deployment_switch_test.go +++ b/services/core/internal/execution/sandbox_deployment_switch_test.go @@ -18,8 +18,8 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -62,7 +62,7 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { case <-time.After(time.Second): t.Fatal("switch drain blocked") } - config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} if err := m.activateDeployment(t.Context(), deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"}); err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { func TestSandboxManagerFailedActivationStaysPaused(t *testing.T) { id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -100,8 +100,8 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -129,7 +129,7 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { m.mu.Lock() originalDrain := m.switchDrained m.mu.Unlock() - config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + config = &RuntimeProvider{InstallationID: id, ProviderKind: "e2b", Mode: "direct", Generation: 2, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("b", 64), Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)} expected := deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b"} ctx, cancel = context.WithTimeout(t.Context(), 10*time.Millisecond) if err := m.activateDeployment(ctx, expected); !errors.Is(err, context.DeadlineExceeded) { @@ -158,7 +158,7 @@ func TestSandboxActivationCannotBypassOutstandingDrain(t *testing.T) { m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, func(_ context.Context, setup deployment.Setup) (PreparedRuntimeDeployment, error) { - return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}}, nil + return PreparedRuntimeDeployment{Config: &RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)}}, nil })) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/sandbox_generations_test.go b/services/core/internal/execution/sandbox_generations_test.go index 32fc48961..733d28f82 100644 --- a/services/core/internal/execution/sandbox_generations_test.go +++ b/services/core/internal/execution/sandbox_generations_test.go @@ -29,7 +29,7 @@ func TestE2BReplacementVerifiesTwiceAndNeverPublishesFailedCommit(t *testing.T) } hub := node.NewHub(node.HubOptions{}) defer hub.Close() - provider := hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1) + provider := hub.Proxy(uuid.NewString(), docker.Operations(), 1) verifyCalls, published, fenced, released := 0, 0, 0, 0 var rejectAt int var rejection error = sandbox.ErrCredentialOwnership diff --git a/services/core/internal/execution/sandbox_provider_contract_test.go b/services/core/internal/execution/sandbox_provider_contract_test.go index 531b3fbbc..83d0cdef2 100644 --- a/services/core/internal/execution/sandbox_provider_contract_test.go +++ b/services/core/internal/execution/sandbox_provider_contract_test.go @@ -18,7 +18,7 @@ func TestSandboxProviderRegistrationDoesNotRequireAnExecutionVendorBranch(t *tes id := uuid.NewString() config := &RuntimeProvider{InstallationID: id, ProviderKind: "contract-fixture", Mode: mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: &lifecycleOnlySandbox{}} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_reset.go b/services/core/internal/execution/sandbox_reset.go index f841d4ee8..241d452b0 100644 --- a/services/core/internal/execution/sandbox_reset.go +++ b/services/core/internal/execution/sandbox_reset.go @@ -64,9 +64,6 @@ func (m *runtimeManager) committedView(ctx context.Context) (deployment.View, er // enter m.active, hold a Session/deployment transaction, or call a provider while // waiting for a deployment drain. Each page has both a row and time bound. func (m *runtimeManager) resetStep(parent context.Context) error { - if m.loadDeployment == nil { - return nil - } ctx, cancel := context.WithTimeout(parent, 5*time.Second) defer cancel() return m.resetPage(parent, ctx) diff --git a/services/core/internal/execution/sandbox_reset_test.go b/services/core/internal/execution/sandbox_reset_test.go index 9ec0bb050..439b16dcb 100644 --- a/services/core/internal/execution/sandbox_reset_test.go +++ b/services/core/internal/execution/sandbox_reset_test.go @@ -100,8 +100,8 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)}, nil + }, unusedPreparation(t)) m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), config) if err != nil { t.Fatal(err) @@ -214,8 +214,8 @@ func TestSandboxResetPublishesCommittedGenerationWithoutReading(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)}, nil + }, unusedPreparation(t)) var published []uint64 config.PublishUnconfigured = func(generation uint64) { if committedReads != 0 { @@ -255,10 +255,10 @@ func TestSandboxResetPublishesCommittedGenerationWithoutReading(t *testing.T) { func TestCommittedResetViewStopsOwnerWithoutLease(t *testing.T) { id := uuid.NewString() reader := &strictDeploymentReader{t: t, snapshot: func(context.Context) (deployment.Snapshot, error) { - return deployment.Snapshot{Record: deployment.Record{InstallationID: id, WebManaged: true, Generation: 1}}, nil + return deployment.Snapshot{Record: deployment.Record{InstallationID: id, Generation: 1}}, nil }} deployments, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) - m, err := newRuntimeManager(Owner{Lease: lostLease{}, Deployment: operations}, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(Owner{Lease: lostLease{}, Deployment: operations}, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -280,7 +280,7 @@ func TestCommittedResetViewStopsOwnerWithoutLease(t *testing.T) { func TestSandboxResetChangesReturnViewReadAfterCommit(t *testing.T) { owner, deployments, reader := resetManager(t) id := initializeE2BDeployment(t, owner) - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_snapshot_budget_test.go b/services/core/internal/execution/sandbox_snapshot_budget_test.go index 467596e9a..eba65917a 100644 --- a/services/core/internal/execution/sandbox_snapshot_budget_test.go +++ b/services/core/internal/execution/sandbox_snapshot_budget_test.go @@ -72,8 +72,8 @@ func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), docker.Operations(), 1)}, nil + }, unusedPreparation(t)) m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), configuration) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 137dd1a82..839009390 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,7 +9,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -85,19 +84,12 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W worker.runtimes.stop() } }() - } - var process *deployment.ProcessDeployment - if worker.runtimes != nil && worker.runtimes.loadDeployment == nil { - config := worker.runtimes.config - process = &deployment.ProcessDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, AdmissionPaused: config.AdmissionPaused} - } - if worker.runtimes != nil && worker.runtimes.loadDeployment != nil { err = owner.Deployment.Claim(ctx, worker.runtimes.setupInstallationID) if err == nil { _, err = worker.runtimes.ensureDeployment(ctx) } } else { - err = owner.Deployment.ConfigureProcess(ctx, process) + err = owner.Deployment.RequireUnclaimed(ctx) } if err != nil { return nil, err diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 08e2569cb..a6ae895e1 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -540,19 +540,3 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A } return loadActivity(ctx, s.pool.Queries(), id) } - -func (s *Store) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - id, err := parseID(installationID) - if err != nil { - return 0, err - } - return s.pool.Queries().CountRuntimeComputeReservations(ctx, id) -} - -func (s *Store) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - id, err := parseID(installationID) - if err != nil { - return 0, err - } - return s.pool.Queries().CountRuntimeRetainedAllocations(ctx, id) -} diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go index b310e6688..84eed9174 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/records.go +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -59,8 +59,8 @@ func translate(err error) error { // credentialcrypto.ErrUnavailable; a credential the key cannot open or // authenticate is an internal decryption error. func record(d sqlc.RuntimeDeployment, cipher *credentialcrypto.Cipher, open bool) deployment.Record { - r := deployment.Record{InstallationID: uuidString(d.InstallationID), WebManaged: d.WebManaged, Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, - Generation: uint64(d.Generation), OwnerEpoch: uint64(d.OwnerEpoch), Mode: d.Mode, AdmissionPaused: d.AdmissionPaused, LocalNodeID: uuidString(d.LocalNodeID), + r := deployment.Record{InstallationID: uuidString(d.InstallationID), Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, + Generation: uint64(d.Generation), OwnerEpoch: uint64(d.OwnerEpoch), Mode: d.Mode, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Specification: d.Specification, Configuration: sandbox.ConfigurationRecord{Public: d.ProviderConfig, Metadata: d.ProviderMetadata}, CredentialStored: len(d.ProviderCredential) > 0} if r.CredentialStored && open { diff --git a/services/core/internal/persistence/postgres/deploymentpg/reset_test.go b/services/core/internal/persistence/postgres/deploymentpg/reset_test.go index 9c46bb524..5e242387a 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/reset_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/reset_test.go @@ -125,8 +125,8 @@ func TestResetDeadlineAndCancel(t *testing.T) { if err := changes.CancelReset(admin(t), installation, view.Generation); err != nil { t.Fatal(err) } - if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE admission_paused OR reset_clear IS NOT NULL OR reset_audit IS NOT NULL"); paused != 0 { - t.Fatal("cancellation left the reset or paused admission") + if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE reset_clear IS NOT NULL OR reset_audit IS NOT NULL"); paused != 0 { + t.Fatal("cancellation left the reset") } if got, want := resetAudit(t, f), []string{"reset_start fixture-admin", "reset_deadline fixture-admin", "reset_cancel fixture-admin"}; !slices.Equal(got, want) { t.Fatalf("audit = %q, want %q", got, want) @@ -146,7 +146,7 @@ func TestResetWritesNothingWithoutAuditOrLease(t *testing.T) { if err := closed.StartReset(admin(t), installation, request); !errors.Is(err, pgunit.ErrLeaseClosed) { t.Fatalf("StartReset on a closed lease = %v", err) } - if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE admission_paused OR reset_clear IS NOT NULL"); paused != 0 { + if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE reset_clear IS NOT NULL"); paused != 0 { t.Fatal("a rejected reset paused admission") } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go index f9ac218b8..6fee76efc 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "strings" "sync" "testing" @@ -112,32 +111,20 @@ func TestSandboxDeploymentSetupConcurrentSelection(t *testing.T) { } } -func TestSandboxDeploymentSetupRejectsFileManagedAndUnleasedWrites(t *testing.T) { +func TestSandboxDeploymentSetupRejectsUnleasedWrites(t *testing.T) { f := newFixture(t) - input := sandbox.Selection{DeploymentSpec: testSpecification("docker"), Provider: "docker"} - process := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), ProviderKind: "docker", - LocalNodeID: uuid.NewString(), LocalCredentialSHA256: setupDigest("local-node-credential"), LocalMaxActive: 4, LocalMaxRetained: 16} + id := uuid.NewString() // Deployment changes run only on the execution lease; a closed one writes nothing. closed := setupClosedExecution(t, f) - if err := closed.ConfigureProcess(t.Context(), &process); !errors.Is(err, pgunit.ErrLeaseClosed) { - t.Fatal("unleased process configuration accepted", err) + if err := closed.Claim(t.Context(), id); !errors.Is(err, pgunit.ErrLeaseClosed) { + t.Fatal("unleased claim accepted", err) } - if _, err := closed.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, pgunit.ErrLeaseClosed) { + if _, err := closed.Initialize(t.Context(), id, sandbox.Selection{DeploymentSpec: testSpecification("docker"), Provider: "docker"}); !errors.Is(err, pgunit.ErrLeaseClosed) { t.Fatal("unleased setup accepted", err) } if view, err := f.service.View(t.Context()); err != nil || view.InstallationID != "" || view.Provider != "" { t.Fatal("unleased writes changed the deployment", view, err) } - changes, _ := f.execution(t) - if err := changes.ConfigureProcess(t.Context(), &process); err != nil { - t.Fatal(err) - } - if _, err := changes.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file-managed deployment changed", err) - } - if err := changes.Claim(t.Context(), process.InstallationID); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file-managed deployment adopted", err) - } } func TestSandboxSelectionRejectsWhitespaceInE2BCredential(t *testing.T) { @@ -183,54 +170,3 @@ func TestSandboxE2BEndpointPersistenceAndOnlineSwitch(t *testing.T) { t.Fatal("online endpoint switch failed", changed, err) } } - -func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) { - f := newFixture(t) - old := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} - if err := setupClosedExecution(t, f).ConfigureProcess(t.Context(), &old); !errors.Is(err, pgunit.ErrLeaseClosed) { - t.Fatal("unleased configuration accepted", err) - } - changes, _ := f.execution(t) - configure := func(selected *deployment.ProcessDeployment) { - t.Helper() - if err := changes.ConfigureProcess(t.Context(), selected); err != nil { - t.Fatal(err) - } - } - configure(&old) - for _, changeID := range []bool{false, true} { - next := old - if changeID { - next.InstallationID = uuid.NewString() - } else { - next.BackendFingerprint = strings.Repeat("b", 64) - } - next.AdmissionPaused = true - if err := changes.ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "maintenance") { - t.Fatal("identity changed before prior maintenance", err) - } - } - old.AdmissionPaused = true - configure(&old) - next := old - next.BackendFingerprint = strings.Repeat("b", 64) - next.AdmissionPaused = false - if err := changes.ConfigureProcess(t.Context(), &next); err == nil { - t.Fatal("switch reopened creation in same operation") - } - next.AdmissionPaused = true - configure(&next) - var id, fingerprint string - var maintenance bool - if err := f.pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,admission_paused FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { - t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) - } - configure(nil) - // Disabling the configured adapter must not forget the old maintenance state. - next.AdmissionPaused = false - configure(&next) - another := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), AdmissionPaused: true} - if err := changes.ConfigureProcess(t.Context(), &another); err == nil { - t.Fatal("nil selection erased the maintenance prerequisite") - } -} diff --git a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go index f739faaec..304b9121c 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go @@ -16,7 +16,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" ) -func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *testing.T) { +func TestSandboxSpecificationRoundTripAndClaimStays(t *testing.T) { for _, provider := range []string{"docker", "microsandbox", "e2b"} { t.Run(provider, func(t *testing.T) { f := newFixture(t) @@ -44,11 +44,8 @@ func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *test if _, err := changes.Initialize(t.Context(), view.InstallationID, changed); !errors.Is(err, deployment.ErrConflict) { t.Fatal("initial setup silently resized a configured deployment", err) } - file := deployment.ProcessDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, AdmissionPaused: true} - for _, candidate := range []*deployment.ProcessDeployment{nil, &file} { - if err := changes.ConfigureProcess(t.Context(), candidate); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file configuration replaced database ownership", err) - } + if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("an owner without sandbox runtimes started on a claimed deployment", err) } after, err := f.service.View(t.Context()) if err != nil || !reflect.DeepEqual(after, view) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 6145dd13b..e0952eb19 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -297,25 +297,6 @@ func (t *deploymentTx) ClaimInstallation(installationID string) error { return t.q.ClaimWebSandboxDeployment(t.ctx, id) } -func (t *deploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { - id, err := parseID(installationID) - if err != nil { - return err - } - return t.q.SetRuntimeDeployment(t.ctx, sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: backendFingerprint, AdmissionPaused: admissionPaused}) -} - -func (t *deploymentTx) SetManagerDeployment(provider, localNodeID string) error { - var local pgtype.UUID - if localNodeID != "" { - var err error - if local, err = parseID(localNodeID); err != nil { - return err - } - } - return t.q.SetRuntimeManagerDeployment(t.ctx, sqlc.SetRuntimeManagerDeploymentParams{ProviderKind: provider, LocalNodeID: local}) -} - func (t *deploymentTx) SaveSelection(selection deployment.SelectionRecord) error { if selection.Generation > math.MaxInt64 { return deployment.ErrInvalidInput diff --git a/services/core/internal/persistence/postgres/placementpg/placementpg.go b/services/core/internal/persistence/postgres/placementpg/placementpg.go index f67cbf294..73fc46b35 100644 --- a/services/core/internal/persistence/postgres/placementpg/placementpg.go +++ b/services/core/internal/persistence/postgres/placementpg/placementpg.go @@ -25,8 +25,7 @@ func LockDeployment(ctx context.Context, q *sqlc.Queries) (placement.Deployment, } return placement.Deployment{ InstallationID: uuidString(d.InstallationID), Provider: d.ProviderKind, Mode: d.Mode, - Generation: uint64(d.Generation), WebManaged: d.WebManaged, AdmissionPaused: d.AdmissionPaused, - Resetting: d.ResetClear.Valid, Specification: d.Specification, + Generation: uint64(d.Generation), Resetting: d.ResetClear.Valid, Specification: d.Specification, }, nil } diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index 65ca06140..1552275b8 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -473,8 +473,8 @@ func TestEnvironmentCreationReservesItsInitialInput(t *testing.T) { } } -// Hosted creation checks admission on the locked deployment, so it sees -// maintenance committed while it waited, and places after creating the +// Hosted creation checks admission on the locked deployment, so it sees a +// reset committed while it waited, and places after creating the // Environment, rolling both back when no node is available. A retry admits // nothing and still returns its Session. func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { @@ -505,19 +505,20 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { done <- err }() awaitBlocked(ctx, t, pool, holder) - if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, admission_paused=true", uuid.New(), strings.Repeat("a", 64)); err != nil { + if _, err := tx.Exec(ctx, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, provider_kind='docker', mode='nodes', generation=1, + reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`, uuid.New(), strings.Repeat("a", 64)); err != nil { t.Fatal(err) } if err := tx.Commit(ctx); err != nil { t.Fatal(err) } - if err := <-done; !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("creation bypassed committed maintenance", err) + if err := <-done; !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("creation bypassed the committed reset", err) } if retry, err := service.CreateSession(ctx, tenant, hosted("existing")); err != nil || retry.Created || retry.Session.ID != existing.Session.ID { t.Fatal("retry ran admission", retry, err) } - exec(t, pool, "UPDATE runtime_deployment SET admission_paused=false, web_managed=true") + exec(t, pool, "UPDATE runtime_deployment SET reset_clear=NULL, reset_requested_at=NULL, reset_forced_at=NULL, reset_audit=NULL, provider_kind='', mode=''") if _, err := service.CreateSession(ctx, tenant, hosted("unplaced")); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("placement without a node", err) } diff --git a/services/core/internal/providercontract/operations.go b/services/core/internal/providercontract/operations.go index ab1aa7a32..cb24ac2f5 100644 --- a/services/core/internal/providercontract/operations.go +++ b/services/core/internal/providercontract/operations.go @@ -5,7 +5,6 @@ package providercontract import ( "errors" "fmt" - "reflect" "regexp" ) @@ -64,32 +63,3 @@ func Require(p Declared, operation string) error { } return p.ProviderOperations()[operation].Check(operation) } - -// Validate checks the existing small interfaces, including methods declared -// unsupported. Every adapter must explicitly implement those rejections. -// A new method cannot be silently covered by an old declaration or stub. -func Validate(p Declared, contracts ...reflect.Type) error { - if p == nil { - return ErrContract - } - value := reflect.ValueOf(p) - if value.Kind() == reflect.Pointer && value.IsNil() { - return ErrContract - } - operations := p.ProviderOperations() - for _, contract := range contracts { - if !value.Type().Implements(contract) { - return fmt.Errorf("%w: missing %s implementation", ErrContract, contract.Name()) - } - for i := 0; i < contract.NumMethod(); i++ { - name := contract.Method(i).Name - if name == "ProviderOperations" { - continue - } - if err := operations[name].Check(name); err != nil && !errors.Is(err, ErrUnsupported) { - return err - } - } - } - return nil -} diff --git a/services/core/internal/runtimeobs/exporter_independence_test.go b/services/core/internal/runtimeobs/exporter_independence_test.go index 2c3b92078..3a24a74ec 100644 --- a/services/core/internal/runtimeobs/exporter_independence_test.go +++ b/services/core/internal/runtimeobs/exporter_independence_test.go @@ -10,7 +10,7 @@ func TestExporterOutageDoesNotDelayOtherDestinations(t *testing.T) { target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} blocked := &gatedExporter{started: make(chan struct{}, 1), release: make(chan struct{})} records := make(chan ExportRecord, 1) - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now}}}, + service, err := NewService(fixedResolver{target: target}, sourceOf(&fixedSource{sample: Sample{ObservedAt: now}}), WithExporter(blocked, ExportOptions{QueueCapacity: 1, Timeout: time.Second}), WithExporter(channelExporter{records: records}, ExportOptions{QueueCapacity: 1, Timeout: time.Second})) if err != nil { diff --git a/services/core/internal/runtimeobs/operations_fixture_test.go b/services/core/internal/runtimeobs/operations_fixture_test.go index d55ebe8c4..8364c7ad0 100644 --- a/services/core/internal/runtimeobs/operations_fixture_test.go +++ b/services/core/internal/runtimeobs/operations_fixture_test.go @@ -2,46 +2,18 @@ package runtimeobs import ( "context" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" ) -func (*fixedSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_has_no_batch_observation"}} -} -func (*fixedSource) ObserveBatch(context.Context, []Target) ([]BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_has_no_batch_observation"} -} -func (blockingSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_has_no_batch_observation"}} -} -func (blockingSource) ObserveBatch(context.Context, []Target) ([]BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_has_no_batch_observation"} -} -func (*countingSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_has_no_batch_observation"}} -} -func (*countingSource) ObserveBatch(context.Context, []Target) ([]BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_has_no_batch_observation"} -} -func (*batchSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, "ObserveBatch": {State: providercontract.Supported}} +func observedOperations() providercontract.Operations { + return providercontract.Operations{"Observe": {State: providercontract.Supported}} } +func (*fixedSource) ProviderOperations() providercontract.Operations { return observedOperations() } +func (blockingSource) ProviderOperations() providercontract.Operations { return observedOperations() } +func (*countingSource) ProviderOperations() providercontract.Operations { return observedOperations() } -func (s *fixedSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } -func (*fixedSource) ObservationProviderType() string { return "fixture" } - -func (s blockingSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } - -func (s *countingSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } -func (*countingSource) ObservationProviderType() string { return "fixture" } - -func (s *batchSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } -func (*batchSource) ObservationProviderType() string { return "fixture" } - -func (s typedSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } - -func (s *failingBatchSource) ResolveObservationSource(context.Context) (Source, error) { return s, nil } - -func (s *unsupportedObservation) ResolveObservationSource(context.Context) (Source, error) { - return s, nil +// sourceOf selects one fixed docker source for every page. +func sourceOf(source Source) func(context.Context) (Source, string, error) { + return func(context.Context) (Source, string, error) { return source, "docker", nil } } diff --git a/services/core/internal/runtimeobs/operations_test.go b/services/core/internal/runtimeobs/operations_test.go index 3e66de33c..fc9e242d0 100644 --- a/services/core/internal/runtimeobs/operations_test.go +++ b/services/core/internal/runtimeobs/operations_test.go @@ -1,69 +1,20 @@ package runtimeobs import ( - "context" - "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "testing" - "time" -) -type failingBatchSource struct { - *fixedSource - batchErr error - batches int -} - -func (*failingBatchSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, "ObserveBatch": {State: providercontract.Supported}} -} -func (s *failingBatchSource) ObserveBatch(context.Context, []Target) ([]BatchResult, error) { - s.batches++ - return nil, s.batchErr -} -func TestBatchFallbackRequiresExplicitSafeUnsupported(t *testing.T) { - for _, test := range []struct { - name string - err error - fallback bool - }{ - {"unsupported", &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "native_batch_not_supported"}, true}, - {"unavailable", ErrUnavailable, false}, - {"timeout", context.DeadlineExceeded, false}, - {"failure", errors.New("provider failed"), false}, - {"bare unsupported", providercontract.ErrUnsupported, false}, - {"unsafe unsupported", &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "private endpoint / key"}, false}, - {"wrong operation", &providercontract.UnsupportedError{Operation: "Observe", Reason: "not_supported"}, false}, - } { - t.Run(test.name, func(t *testing.T) { - now := time.Now() - ratio := 0.5 - source := &failingBatchSource{fixedSource: &fixedSource{sample: Sample{ObservedAt: now, CPUUtilizationRatio: &ratio}}, batchErr: test.err} - target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) - if err != nil { - t.Fatal(err) - } - observations, errs := service.ObserveSessions(t.Context(), []SessionIdentity{{TenantID: "tenant", SessionID: "session"}}, PageOptions{}) - if source.batches != 1 || (source.calls == 1) != test.fallback { - t.Fatalf("batch=%d single=%d", source.batches, source.calls) - } - if test.fallback && (errs[0] != nil || observations[0].Status != StatusObserved) { - t.Fatal(observations, errs) - } - }) - } -} + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" +) type unsupportedObservation struct{ *fixedSource } func (*unsupportedObservation) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": {State: providercontract.Supported}, "ObservationProviderType": {State: providercontract.Supported}, "Observe": {State: providercontract.Unsupported, Reason: "native_metrics_not_supported"}, "ObserveBatch": {State: providercontract.Unsupported, Reason: "native_metrics_not_supported"}} + return providercontract.Operations{"Observe": {State: providercontract.Unsupported, Reason: "native_metrics_not_supported"}} } func TestUnsupportedObservationIsNotUnavailable(t *testing.T) { source := &unsupportedObservation{&fixedSource{}} target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err := NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/runtimeobs/sampler.go b/services/core/internal/runtimeobs/sampler.go index b55e205db..a89384344 100644 --- a/services/core/internal/runtimeobs/sampler.go +++ b/services/core/internal/runtimeobs/sampler.go @@ -27,8 +27,7 @@ type SessionLister interface { ListRuntimeObservationSessions(context.Context, string, int) (SessionPage, error) } -// HistoryObserver reads one listed page, so providers with a batch read can -// sample the whole page in one bounded request. +// HistoryObserver reads one listed page with bounded concurrency. type HistoryObserver interface { ObserveSessionsForHistory(context.Context, []SessionIdentity, OwnershipChecker, PageOptions) ([]Observation, []error) } diff --git a/services/core/internal/runtimeobs/sampler_test.go b/services/core/internal/runtimeobs/sampler_test.go index 2f1e119e6..da14b77b4 100644 --- a/services/core/internal/runtimeobs/sampler_test.go +++ b/services/core/internal/runtimeobs/sampler_test.go @@ -146,7 +146,7 @@ func TestSamplerSweepsEveryPageAndIsolatesSessionFailures(t *testing.T) { func TestSamplerBoundsConcurrencyAndSourceDeadline(t *testing.T) { source := &countingSource{} target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err := NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } @@ -246,7 +246,7 @@ func TestSamplerPreservesProviderTimeoutAndFinalFenceAfterSlowResolution(t *test records := make(chan ExportRecord, 1) service, err := NewService( resolver, - map[string]SourceResolver{"provider": blockingSource{}}, + sourceOf(blockingSource{}), WithExporter(channelExporter{records: records}, ExportOptions{}), ) if err != nil { diff --git a/services/core/internal/runtimeobs/service.go b/services/core/internal/runtimeobs/service.go index 101c6fe9f..c38db7359 100644 --- a/services/core/internal/runtimeobs/service.go +++ b/services/core/internal/runtimeobs/service.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "reflect" "sync" "time" @@ -30,12 +29,16 @@ const ( type Service struct { resolver TargetResolver - sources map[string]SourceResolver + source func(context.Context) (Source, string, error) now func() time.Time exports []*exportDispatcher } -func NewService(resolver TargetResolver, sources map[string]SourceResolver, options ...ServiceOption) (*Service, error) { +// NewService reads managed Runtimes through source, which returns the Sandbox +// Provider of the deployment's current selection and its registered kind, or +// ErrUnavailable while none is selected. A nil source means this Core has no +// managed deployment. +func NewService(resolver TargetResolver, source func(context.Context) (Source, string, error), options ...ServiceOption) (*Service, error) { if resolver == nil { return nil, errors.New("Runtime observation resolver is required") } @@ -48,20 +51,7 @@ func NewService(resolver TargetResolver, sources map[string]SourceResolver, opti return nil, err } } - copySources := make(map[string]SourceResolver, len(sources)) - for key, source := range sources { - if key == "" || source == nil { - return nil, errors.New("invalid Runtime observation source") - } - if err := providercontract.Validate(source, reflect.TypeFor[SourceResolver]()); err != nil { - return nil, err - } - if err := providercontract.Require(source, "ResolveObservationSource"); err != nil { - return nil, fmt.Errorf("%w: observation source resolution must be supported", providercontract.ErrContract) - } - copySources[key] = source - } - service := &Service{resolver: resolver, sources: copySources, now: time.Now} + service := &Service{resolver: resolver, source: source, now: time.Now} for _, export := range config.exporters { service.exports = append(service.exports, newExportDispatcher(export.exporter, export.exportOptions)) } @@ -109,14 +99,12 @@ func (s *Service) ObserveSessionForHistory(ctx context.Context, tenantID, sessio type PageOptions struct { // Concurrency bounds identity resolution and per-target provider reads. Concurrency int - // SourceTimeout bounds each provider read, including one batch read. + // SourceTimeout bounds each provider read. SourceTimeout time.Duration } -// ObserveSessions observes one page of Sessions. Running targets of a source -// that explicitly supports BatchSource share one provider read per MaxBatchTargets; -// other sources are read per target, exactly as ObserveSession reads them. -// Results and errors are aligned with sessions. +// ObserveSessions observes one page of Sessions, reading each running target +// exactly as ObserveSession reads it. Results and errors are aligned with sessions. func (s *Service) ObserveSessions(ctx context.Context, sessions []SessionIdentity, options PageOptions) ([]Observation, []error) { return s.observeSessions(ctx, sessions, CollectionSourceOnRead, nil, options) } @@ -141,8 +129,6 @@ func (s *Service) observeSession(ctx context.Context, tenantID, sessionID string // sourceRead is a resolved running managed target awaiting its provider sample. type sourceRead struct { index int - key string - source Source target Target providerType string } @@ -162,117 +148,39 @@ func (s *Service) observeSessions(ctx context.Context, sessions []SessionIdentit } observations[index], errs[index] = observation, err }) - // A provider key selects one source; keep page order within each group. - var keys []string - groups := map[string][]*sourceRead{} + var pending []*sourceRead for _, read := range reads { - if read == nil { - continue - } - if _, ok := groups[read.key]; !ok { - keys = append(keys, read.key) - } - groups[read.key] = append(groups[read.key], read) - } - for _, key := range keys { - group := groups[key] - sourceCtx, stop := sourceContext(ctx, options.SourceTimeout) - source, err := s.sources[key].ResolveObservationSource(sourceCtx) - stop() - if err == nil { - err = ValidateSource(source) - } - if err != nil { - for _, read := range group { - observations[read.index], errs[read.index] = s.complete(ctx, read, Sample{}, err, 0, collectionSource, owner) - } - continue - } - providerType := source.ObservationProviderType() - for _, read := range group { - read.source, read.providerType = source, providerType - } - for start := 0; start < len(group); start += MaxBatchTargets { - chunk := group[start:min(start+MaxBatchTargets, len(group))] - if s.readBatch(ctx, chunk, observations, errs, collectionSource, owner, options.SourceTimeout) { - continue - } - parallel(len(chunk), options.Concurrency, func(index int) { - read := chunk[index] - sourceCtx, stop := sourceContext(ctx, options.SourceTimeout) - started := time.Now() - var sample Sample - err := providercontract.Require(read.source, "Observe") - if err == nil { - sample, err = read.source.Observe(sourceCtx, read.target) - } - stop() - observations[read.index], errs[read.index] = s.complete(ctx, read, sample, err, time.Since(started), collectionSource, owner) - }) - } - } - return observations, errs -} - -// readBatch reports false, without results, when the source has no batch read -// for its current provider. -func (s *Service) readBatch(ctx context.Context, chunk []*sourceRead, observations []Observation, errs []error, collectionSource CollectionSource, owner OwnershipChecker, sourceTimeout time.Duration) bool { - batch, ok := chunk[0].source.(BatchSource) - if !ok { // NewService rejects this; never treat malformed registration as unsupported. - for _, read := range chunk { - errs[read.index] = providercontract.ErrContract - } - return true - } - if err := providercontract.Require(chunk[0].source, "ObserveBatch"); err != nil { - if errors.Is(err, providercontract.ErrUnsupported) { - return false - } - for _, read := range chunk { - errs[read.index] = err + if read != nil { + pending = append(pending, read) } - return true } - targets := make([]Target, len(chunk)) - for index, read := range chunk { - targets[index] = read.target + if len(pending) == 0 { + return observations, errs } - // One batch read replaces up to MaxBatchTargets single reads, so it may take - // longer than one of them without exceeding the page's overall cost. - if sourceTimeout > 0 { - sourceTimeout = max(sourceTimeout, minBatchSourceTimeout) - } - sourceCtx, stop := sourceContext(ctx, sourceTimeout) - started := time.Now() - results, batchErr := batch.ObserveBatch(sourceCtx, targets) + // One source serves the whole page. + sourceCtx, stop := sourceContext(ctx, options.SourceTimeout) + source, providerType, err := s.source(sourceCtx) stop() - if _, unsupported := providercontract.UnsupportedReason(batchErr, "ObserveBatch"); unsupported { - return false - } - if errors.Is(batchErr, providercontract.ErrUnsupported) { - batchErr = providercontract.ErrContract - } - if batchErr != nil { - for _, read := range chunk { - observations[read.index], errs[read.index] = s.complete(ctx, read, Sample{}, batchErr, 0, collectionSource, owner) + if err != nil { + for _, read := range pending { + observations[read.index], errs[read.index] = s.complete(ctx, read, Sample{}, err, 0, collectionSource, owner) } - return true + return observations, errs } - duration := time.Since(started) - for index, read := range chunk { - if len(results) != len(chunk) { - errs[read.index] = errors.New("Runtime observation batch returned mismatched results") - continue - } - // The rows share one provider read; only the first carries its duration, - // so sample-duration telemetry counts each read once. - rowDuration := time.Duration(0) - if index == 0 { - rowDuration = duration + parallel(len(pending), options.Concurrency, func(index int) { + read := pending[index] + read.providerType = providerType + sourceCtx, stop := sourceContext(ctx, options.SourceTimeout) + started := time.Now() + var sample Sample + err := providercontract.Require(source, "Observe") + if err == nil { + sample, err = source.Observe(sourceCtx, read.target) } - observations[read.index], errs[read.index] = s.complete(ctx, read, results[index].Sample, results[index].Err, rowDuration, collectionSource, owner) - } - return true + stop() + observations[read.index], errs[read.index] = s.complete(ctx, read, sample, err, time.Since(started), collectionSource, owner) + }) + return observations, errs } // resolve returns either a finished observation or a pending provider read. @@ -317,11 +225,10 @@ func (s *Service) resolve(ctx context.Context, tenantID, sessionID string, owner default: return Observation{}, nil, errors.New("invalid managed Runtime allocation state") } - _, ok := s.sources[target.Instance.ProviderKey] - if !ok { + if s.source == nil { return Observation{Target: target, Status: StatusUnavailable, Reason: "source_not_configured", ResolvedAt: resolvedAt}, nil, nil } - return Observation{}, &sourceRead{key: target.Instance.ProviderKey, target: target}, nil + return Observation{}, &sourceRead{target: target}, nil } // complete classifies one provider result and hands it to history export. @@ -352,8 +259,6 @@ func (s *Service) complete(ctx context.Context, read *sourceRead, sample Sample, return s.finish(ctx, observation, collectionSource, owner) } -const minBatchSourceTimeout = 5 * time.Second - func sourceContext(ctx context.Context, timeout time.Duration) (context.Context, context.CancelFunc) { if timeout > 0 { return context.WithTimeout(ctx, timeout) diff --git a/services/core/internal/runtimeobs/service_test.go b/services/core/internal/runtimeobs/service_test.go index ca3d9dbc2..763b4363d 100644 --- a/services/core/internal/runtimeobs/service_test.go +++ b/services/core/internal/runtimeobs/service_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "math" "sync" "testing" @@ -38,13 +37,6 @@ func (s *fixedSource) Observe(context.Context, Target) (Sample, error) { return s.sample, s.err } -type typedSource struct { - *fixedSource - providerType string -} - -func (s typedSource) ObservationProviderType() string { return s.providerType } - type blockingSource struct{} func (blockingSource) Observe(ctx context.Context, _ Target) (Sample, error) { @@ -52,8 +44,6 @@ func (blockingSource) Observe(ctx context.Context, _ Target) (Sample, error) { return Sample{}, ctx.Err() } -func (blockingSource) ObservationProviderType() string { return "docker" } - type channelExporter struct { records chan ExportRecord err error @@ -125,7 +115,7 @@ func TestServiceDoesNotCallSourcesForUnsupportedModes(t *testing.T) { if mode == ModeSelfHosted { target.EnvironmentID = "environment" } - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err := NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } @@ -151,7 +141,7 @@ func TestServicePreservesUnavailableAndObservedZero(t *testing.T) { zeroMemory := uint64(0) now := time.Date(2026, 9, 22, 1, 0, 0, 0, time.UTC) source := &fixedSource{sample: Sample{ObservedAt: now, CPUUsageSecondsTotal: &zeroCPU, MemoryUsageBytes: &zeroMemory}} - service, err = NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err = NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } @@ -176,13 +166,13 @@ func TestServiceExportsOnlySanitizedValidatedRecords(t *testing.T) { ProviderState: json.RawMessage(`{"native_id":"must-not-export"}`), }, } - source := typedSource{fixedSource: &fixedSource{sample: Sample{ + source := &fixedSource{sample: Sample{ ObservedAt: now, StartedAt: &startedAt, CPUUsageSecondsTotal: &cpuSeconds, CPUCapacityCores: &cpuCapacity, MemoryUsageBytes: &memoryUsage, MemoryLimitBytes: &memoryLimit, - }}, providerType: "docker"} + }} records := make(chan ExportRecord, 1) - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}, WithExporter(channelExporter{records: records}, ExportOptions{QueueCapacity: 1, Timeout: time.Second})) + service, err := NewService(fixedResolver{target: target}, sourceOf(source), WithExporter(channelExporter{records: records}, ExportOptions{QueueCapacity: 1, Timeout: time.Second})) if err != nil { t.Fatal(err) } @@ -224,7 +214,7 @@ func TestServiceMarksPeriodicHistoryCollection(t *testing.T) { records := make(chan ExportRecord, 1) service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now, StartedAt: &startedAt}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now, StartedAt: &startedAt}}), WithExporter(channelExporter{records: records}, ExportOptions{}), ) if err != nil { @@ -257,7 +247,7 @@ func TestServiceDoesNotExportPeriodicSampleAfterOwnershipLoss(t *testing.T) { records := make(chan ExportRecord, 1) service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now, StartedAt: &startedAt}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now, StartedAt: &startedAt}}), WithExporter(channelExporter{records: records}, ExportOptions{}), ) if err != nil { @@ -278,40 +268,13 @@ func TestServiceDoesNotExportPeriodicSampleAfterOwnershipLoss(t *testing.T) { } } -func TestServiceRejectsUnsafeProviderTypeBeforeSamplingOrExport(t *testing.T) { - now := time.Date(2026, 9, 22, 1, 0, 0, 0, time.UTC) - target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - source := typedSource{fixedSource: &fixedSource{sample: Sample{ObservedAt: now}}, providerType: "docker native_id=secret"} - records := make(chan ExportRecord, 1) - service, err := NewService( - fixedResolver{target: target}, - map[string]SourceResolver{"provider": source}, - WithExporter(channelExporter{records: records}, ExportOptions{}), - ) - if err != nil { - t.Fatal(err) - } - service.now = func() time.Time { return now } - if _, err := service.ObserveSession(t.Context(), "tenant", "session"); err == nil || source.calls != 0 { - t.Fatalf("unsafe provider type reached sampling: err=%v calls=%d", err, source.calls) - } - if err := service.Close(t.Context()); err != nil { - t.Fatal(err) - } - select { - case record := <-records: - t.Fatalf("unsafe provider type reached exporter: %+v", record) - default: - } -} - func TestServiceExportQueueNeverBlocksOrChangesObservation(t *testing.T) { now := time.Date(2026, 9, 22, 1, 0, 0, 0, time.UTC) target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} exporter := &gatedExporter{started: make(chan struct{}, 1), release: make(chan struct{})} service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now}}), WithExporter(exporter, ExportOptions{QueueCapacity: 1, Timeout: time.Second}), ) if err != nil { @@ -370,7 +333,7 @@ func TestServiceIgnoresExporterFailureAndValidatesOptions(t *testing.T) { target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now}}), WithExporter(channelExporter{records: records, err: errors.New("backend unavailable")}, ExportOptions{}), ) if err != nil { @@ -392,7 +355,7 @@ func TestServiceCloseHonorsItsDeadlineWhenExporterDoesNot(t *testing.T) { target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now}}), WithExporter(exporter, ExportOptions{QueueCapacity: 1, Timeout: time.Millisecond}), ) if err != nil { @@ -422,7 +385,7 @@ func TestServiceIsolatesExporterPanics(t *testing.T) { target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": &fixedSource{sample: Sample{ObservedAt: now}}}, + sourceOf(&fixedSource{sample: Sample{ObservedAt: now}}), WithExporter(exporter, ExportOptions{}), ) if err != nil { @@ -455,9 +418,7 @@ func TestServiceMapsOnlyDeclaredUnavailability(t *testing.T) { {err: context.DeadlineExceeded, wantReason: "sample_timeout"}, {err: errors.New("Docker permission denied"), wantError: true}, } { - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{ - "provider": typedSource{fixedSource: &fixedSource{err: tc.err}, providerType: "docker"}, - }) + service, err := NewService(fixedResolver{target: target}, sourceOf(&fixedSource{err: tc.err})) if err != nil { t.Fatal(err) } @@ -476,7 +437,7 @@ func TestServiceMapsAnActualSourceDeadlineWithoutLeakingIt(t *testing.T) { EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}, } - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": blockingSource{}}) + service, err := NewService(fixedResolver{target: target}, sourceOf(blockingSource{})) if err != nil { t.Fatal(err) } @@ -496,7 +457,7 @@ func TestServiceExportsPeriodicSourceTimeoutAfterFinalOwnershipFence(t *testing. records := make(chan ExportRecord, 1) service, err := NewService( fixedResolver{target: target}, - map[string]SourceResolver{"provider": blockingSource{}}, + sourceOf(blockingSource{}), WithExporter(channelExporter{records: records}, ExportOptions{}), ) if err != nil { @@ -537,7 +498,7 @@ func TestServiceClassifiesResolverAndTerminalAllocationUnavailability(t *testing source := &fixedSource{} target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "creating"}} - service, err = NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err = NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } @@ -547,7 +508,7 @@ func TestServiceClassifiesResolverAndTerminalAllocationUnavailability(t *testing } target = Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "released"}} - service, err = NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": &fixedSource{}}) + service, err = NewService(fixedResolver{target: target}, sourceOf(&fixedSource{})) if err != nil { t.Fatal(err) } @@ -560,7 +521,7 @@ func TestServiceClassifiesResolverAndTerminalAllocationUnavailability(t *testing target = Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{ AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running", AllocationCreatedAt: time.Now().Add(time.Hour), }} - service, err = NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) + service, err = NewService(fixedResolver{target: target}, sourceOf(source)) if err != nil { t.Fatal(err) } @@ -584,7 +545,7 @@ func TestServiceRejectsUnsafeProviderSamples(t *testing.T) { {ObservedAt: now, MemoryUsageBytes: &tooLarge}, {ObservedAt: now, MemoryLimitBytes: &tooLarge}, } { - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": &fixedSource{sample: sample}}) + service, err := NewService(fixedResolver{target: target}, sourceOf(&fixedSource{sample: sample})) if err != nil { t.Fatal(err) } @@ -614,67 +575,6 @@ func TestServiceRejectsMismatchedResolvedOwnership(t *testing.T) { func float64Pointer(value float64) *float64 { return &value } -type batchSource struct { - mu sync.Mutex - batches []int - sample Sample -} - -func (s *batchSource) Observe(context.Context, Target) (Sample, error) { - return Sample{}, errors.New("per-target read used for a batch source") -} - -func (s *batchSource) ObserveBatch(ctx context.Context, targets []Target) ([]BatchResult, error) { - time.Sleep(time.Millisecond) - s.mu.Lock() - s.batches = append(s.batches, len(targets)) - s.mu.Unlock() - results := make([]BatchResult, len(targets)) - for index, target := range targets { - results[index].Sample = s.sample - if target.SessionID == "stopped" { - results[index].Err = ErrNotRunning - } - } - return results, nil -} - -func TestServiceBatchesPageReadsWithinProviderLimit(t *testing.T) { - now := time.Date(2026, 9, 25, 1, 0, 0, 0, time.UTC) - ratio := 0.25 - source := &batchSource{sample: Sample{ObservedAt: now, CPUUtilizationRatio: &ratio}} - target := Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}} - service, err := NewService(fixedResolver{target: target}, map[string]SourceResolver{"provider": source}) - if err != nil { - t.Fatal(err) - } - service.now = func() time.Time { return now } - sessions := make([]SessionIdentity, 150) - for index := range sessions { - sessions[index] = SessionIdentity{TenantID: "tenant", SessionID: fmt.Sprint(index)} - } - sessions[120].SessionID = "stopped" - observations, errs := service.ObserveSessions(t.Context(), sessions, PageOptions{Concurrency: 8, SourceTimeout: time.Second}) - if len(source.batches) != 2 || source.batches[0] != MaxBatchTargets || source.batches[1] != 50 { - t.Fatalf("page was not read in provider-sized batches: %v", source.batches) - } - for index, observation := range observations { - if errs[index] != nil || observation.Target.SessionID != sessions[index].SessionID { - t.Fatalf("batch result %d was not aligned: %+v %v", index, observation, errs[index]) - } - } - if observations[120].Status != StatusUnavailable || observations[120].Reason != "runtime_not_running" || - observations[0].Status != StatusObserved || *observations[0].Sample.CPUUtilizationRatio != .25 { - t.Fatalf("batch results were not classified: %+v %+v", observations[0], observations[120]) - } - // Each provider read reports its duration once, on the first row of its batch. - for index, observation := range observations { - if (observation.SourceDuration > 0) != (index == 0 || index == MaxBatchTargets) { - t.Fatalf("row %d source duration = %v", index, observation.SourceDuration) - } - } -} - func TestSampleWithoutNewerFieldsKeepsItsNodeWireForm(t *testing.T) { observedAt := time.Date(2026, 9, 25, 1, 0, 0, 0, time.UTC) cpu, memory := 1.5, uint64(1024) diff --git a/services/core/internal/runtimeobs/source.go b/services/core/internal/runtimeobs/source.go index 8886a42b1..d29e446bd 100644 --- a/services/core/internal/runtimeobs/source.go +++ b/services/core/internal/runtimeobs/source.go @@ -3,9 +3,6 @@ package runtimeobs import ( "context" "errors" - "fmt" - "reflect" - "regexp" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" ) @@ -15,55 +12,13 @@ var ( ErrNotRunning = errors.New("Runtime is not running") ) -// SourceResolver selects one immutable source for a page of provider reads. -// An unconfigured resolver returns ErrUnavailable. Registration never resolves -// a source; callers validate each selected source before reading it. -type SourceResolver interface { - providercontract.Declared - ResolveObservationSource(context.Context) (Source, error) -} - -// Source reads one provider-owned Runtime instance. Implementations must verify -// ownership before returning data and must not renew, restart, or stop compute. +// Source is the observation half of a Sandbox Provider, which +// services/core/internal/sandbox/sandbox_provider.go defines. type Source interface { providercontract.Declared - // ObservationProviderType is a nonempty, immutable telemetry identity. - ObservationProviderType() string Observe(context.Context, Target) (Sample, error) } -// MaxBatchTargets bounds the targets of one BatchSource read. -const MaxBatchTargets = 100 - -// BatchSource reads multiple instances under the same ownership rules as Observe. -// Unsupported returns a typed providercontract.UnsupportedError before reading; -// only that result permits per-target observation. Unavailable or failed reads -// must not silently retry through Observe. Successful results preserve target order. -type BatchSource interface { - ObserveBatch(context.Context, []Target) (results []BatchResult, err error) -} - -type BatchResult struct { - Sample Sample - Err error -} - type TargetResolver interface { Resolve(context.Context, string, string) (Target, error) } - -var providerTypePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,31}$`) - -// ValidateSource checks every read operation and its immutable provider identity. -func ValidateSource(source Source) error { - if err := providercontract.Validate(source, reflect.TypeFor[Source](), reflect.TypeFor[BatchSource]()); err != nil { - return err - } - if err := providercontract.Require(source, "ObservationProviderType"); err != nil { - return fmt.Errorf("%w: observation identity must be supported", providercontract.ErrContract) - } - if !providerTypePattern.MatchString(source.ObservationProviderType()) { - return fmt.Errorf("%w: invalid Runtime observation provider type", providercontract.ErrContract) - } - return nil -} diff --git a/services/core/internal/runtimeobs/source_test.go b/services/core/internal/runtimeobs/source_test.go index e65753320..655d5a64e 100644 --- a/services/core/internal/runtimeobs/source_test.go +++ b/services/core/internal/runtimeobs/source_test.go @@ -10,78 +10,24 @@ import ( ) type selectingSource struct { - source Source - err error - calls int - support providercontract.Support + source Source + providerType string + err error + calls int } -func (s *selectingSource) ProviderOperations() providercontract.Operations { - return providercontract.Operations{"ResolveObservationSource": s.support} -} -func (s *selectingSource) ResolveObservationSource(context.Context) (Source, error) { +func (s *selectingSource) load(context.Context) (Source, string, error) { s.calls++ - return s.source, s.err -} - -type identityDeclaration struct { - typedSource - support providercontract.Support -} - -func (s identityDeclaration) ProviderOperations() providercontract.Operations { - operations := s.typedSource.ProviderOperations() - operations["ObservationProviderType"] = s.support - return operations -} - -func TestSourceBindingRejectsInvalidIdentityBeforeReadOrExport(t *testing.T) { - for _, test := range []struct { - name, identity string - support providercontract.Support - }{ - {"empty", "", providercontract.Support{State: providercontract.Supported}}, - {"unsafe", "secret:provider", providercontract.Support{State: providercontract.Supported}}, - {"undeclared", "docker", providercontract.Support{}}, - {"unsupported", "docker", providercontract.Support{State: providercontract.Unsupported, Reason: "missing_identity"}}, - } { - t.Run(test.name, func(t *testing.T) { - source := identityDeclaration{typedSource: typedSource{fixedSource: &fixedSource{}, providerType: test.identity}, support: test.support} - resolver := &selectingSource{source: source, support: providercontract.Support{State: providercontract.Supported}} - records := make(chan ExportRecord, 1) - service, err := NewService(observableTarget(), map[string]SourceResolver{"provider": resolver}, WithExporter(channelExporter{records: records}, ExportOptions{})) - if err != nil { - t.Fatal(err) - } - if _, err := service.ObserveSession(t.Context(), "tenant", "session"); !errors.Is(err, providercontract.ErrContract) { - t.Fatalf("invalid identity accepted: %v", err) - } - if source.calls != 0 { - t.Fatal("invalid identity reached provider read") - } - if err := service.Close(t.Context()); err != nil { - t.Fatal(err) - } - if len(records) != 0 { - t.Fatal("invalid identity reached export") - } - }) - } + return s.source, s.providerType, s.err } func observableTarget() fixedResolver { return fixedResolver{target: Target{EnvironmentID: "environment", Mode: ModeManaged, Instance: Instance{AllocationID: "allocation", ProviderKey: "provider", AllocationState: "running"}}} } -func TestSourceResolverRegistrationAndUnavailableSelection(t *testing.T) { - for _, support := range []providercontract.Support{{}, {State: providercontract.Unsupported, Reason: "no_source"}} { - resolver := &selectingSource{support: support} - if _, err := NewService(observableTarget(), map[string]SourceResolver{"provider": resolver}); !errors.Is(err, providercontract.ErrContract) || resolver.calls != 0 { - t.Fatal("invalid resolver registration accepted or performed I/O", err) - } - } - resolver := &selectingSource{err: ErrUnavailable, support: providercontract.Support{State: providercontract.Supported}} - service, err := NewService(observableTarget(), map[string]SourceResolver{"provider": resolver}) +func TestUnavailableAndNilSourceSelection(t *testing.T) { + resolver := &selectingSource{err: ErrUnavailable} + service, err := NewService(observableTarget(), resolver.load) if err != nil || resolver.calls != 0 { t.Fatal("registration resolved unconfigured provider", err) } @@ -89,35 +35,34 @@ func TestSourceResolverRegistrationAndUnavailableSelection(t *testing.T) { if err != nil || observation.Status != StatusUnavailable || observation.Reason != "sample_unavailable" || observation.ProviderType != "" { t.Fatal(observation, err) } - var nilSource *fixedSource - resolver.err, resolver.source = nil, nilSource + resolver.err = nil if _, err := service.ObserveSession(t.Context(), "tenant", "session"); !errors.Is(err, providercontract.ErrContract) { - t.Fatal("typed nil source accepted", err) + t.Fatal("nil source accepted", err) } } type reconfiguringSource struct { - typedSource + *fixedSource resolver *selectingSource next Source } func (s *reconfiguringSource) Observe(ctx context.Context, target Target) (Sample, error) { - s.resolver.source = s.next - return s.typedSource.Observe(ctx, target) + s.resolver.source, s.resolver.providerType = s.next, "next" + return s.fixedSource.Observe(ctx, target) } func TestSourceSelectionStaysBoundForWholePage(t *testing.T) { now := time.Now() - next := typedSource{fixedSource: &fixedSource{sample: Sample{ObservedAt: now}}, providerType: "next"} - resolver := &selectingSource{support: providercontract.Support{State: providercontract.Supported}} - previous := &reconfiguringSource{typedSource: typedSource{fixedSource: &fixedSource{sample: Sample{ObservedAt: now}}, providerType: "previous"}, resolver: resolver, next: next} + next := &fixedSource{sample: Sample{ObservedAt: now}} + resolver := &selectingSource{providerType: "previous"} + previous := &reconfiguringSource{fixedSource: &fixedSource{sample: Sample{ObservedAt: now}}, resolver: resolver, next: next} resolver.source = previous - service, err := NewService(observableTarget(), map[string]SourceResolver{"provider": resolver}) + service, err := NewService(observableTarget(), resolver.load) if err != nil { t.Fatal(err) } - sessions := make([]SessionIdentity, MaxBatchTargets+1) + sessions := make([]SessionIdentity, 3) for index := range sessions { sessions[index] = SessionIdentity{TenantID: "tenant", SessionID: "session"} } diff --git a/services/core/internal/sandbox/configuration.go b/services/core/internal/sandbox/configuration.go index e0fbc5cc6..36ace5baa 100644 --- a/services/core/internal/sandbox/configuration.go +++ b/services/core/internal/sandbox/configuration.go @@ -2,74 +2,11 @@ package sandbox import ( "bytes" - "context" "encoding/json" "io" "slices" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" -) - -// Requirement has no implicit default: registration must choose either value. -type Requirement string - -const ( - Required Requirement = "required" - NotRequired Requirement = "not_required" ) -type ConfigurationRequirements struct { - Credential Requirement - PublicOrigin Requirement - Discovery providercontract.Support -} - -// Configuration is an adapter-owned typed value, never a request or response DTO. -// Implementations must exclude secrets from JSON and safe diagnostic output. -type Configuration interface { - HasCredential() bool - ReplacesCredential() bool -} - -// ConfigurationRecord separates public selectors, read-only observations and -// secret bytes. Store encrypts Secret with the installation and generation. -// Only adapter codecs may produce Public and Metadata; neither is input passthrough. -type ConfigurationRecord struct { - Public json.RawMessage - Metadata json.RawMessage - Secret []byte `json:"-"` -} - -// ConfigurationAdapter owns all interpretation of provider configuration. -// Decode loads retained ownership without remote discovery or new-build admission. -// Normalize validates a candidate; ResolveChange first applies omitted-field -// inheritance, then normalizes. Equal compares normalized identity, excluding -// discovery metadata and explicit credential-submission intent. -type ConfigurationAdapter interface { - Requirements() ConfigurationRequirements - DecodeInput(public, credential json.RawMessage) (Configuration, error) - Encode(Configuration) (ConfigurationRecord, error) - Decode(ConfigurationRecord) (Configuration, error) - Normalize(Selection) (Selection, error) - ResolveChange(next, previous Selection) (Selection, error) - WithCredential(owner, candidate Configuration) (Configuration, error) - Equal(a, b Configuration) (bool, error) -} - -// ConfigurationDiscoveryInput is a transient read-only request. Query is typed -// and validated by the adapter; it cannot select a compute mutation. -type ConfigurationDiscoveryInput struct { - Configuration json.RawMessage `json:"configuration" swaggertype:"object"` - Credential json.RawMessage `json:"credential" swaggertype:"object"` - Query json.RawMessage `json:"query" swaggertype:"object"` -} - -// ConfigurationDiscoverer is separate from compute and candidate admission. -// Support must also be explicitly declared in ConfigurationRequirements. -type ConfigurationDiscoverer interface { - DiscoverConfiguration(context.Context, ConfigurationDiscoveryInput, ProcessPaths) (json.RawMessage, error) -} - // DecodeConfigurationObject rejects unknown fields, null, nonobjects and trailing // input without exposing submitted content in its error. Missing objects are empty. func DecodeConfigurationObject(raw json.RawMessage, target any, allowed ...string) error { diff --git a/services/core/internal/sandbox/configuration_errors.go b/services/core/internal/sandbox/configuration_errors.go deleted file mode 100644 index 7c02d9728..000000000 --- a/services/core/internal/sandbox/configuration_errors.go +++ /dev/null @@ -1,24 +0,0 @@ -package sandbox - -// ConfigurationError is a fixed safe diagnostic, never SDK text or submitted data. -// Class describes the request outcome; it does not authorize replay. -type ConfigurationError struct { - Class ConfigurationErrorClass - Code, Param, Message string -} -type ConfigurationErrorClass string - -const ( - ConfigurationInvalid ConfigurationErrorClass = "invalid" - ConfigurationConflict ConfigurationErrorClass = "conflict" - ConfigurationUnconfirmed ConfigurationErrorClass = "unconfirmed" -) - -func (e *ConfigurationError) Error() string { return e.Message } - -var ( - ErrCredentialRejected = &ConfigurationError{ConfigurationInvalid, "sandbox_credential_invalid", "credential", "The sandbox provider credential was rejected."} - ErrCredentialOwnership = &ConfigurationError{ConfigurationConflict, "sandbox_credential_ownership", "credential", "The credential cannot manage the retained deployment. Reset before changing accounts."} - ErrConfigurationUnconfirmed = &ConfigurationError{ConfigurationUnconfirmed, "sandbox_verification_unconfirmed", "", "Sandbox provider verification could not be confirmed."} - ErrConfigurationSelection = &ConfigurationError{ConfigurationInvalid, "sandbox_configuration_invalid", "configuration", "Select a ready immutable provider configuration with matching resources."} -) diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index c2758008d..2722da0dd 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -3,38 +3,27 @@ package docker import ( "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) -var _ sandbox.SelectionDiscoverer = (*Provider)(nil) -var _ sandbox.CredentialVerifier = (*Provider)(nil) -var _ runtimeobs.BatchSource = (*Provider)(nil) - // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Supported}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "docker_does_not_support_batch_observation"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "docker_does_not_support_selection_discovery"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "docker_does_not_support_credential_verification"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "Suspend": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "Resume": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } @@ -65,12 +54,3 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } -func (p *Provider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: Operations()["ObserveBatch"].Reason} -} -func (p *Provider) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: Operations()["DiscoverSelection"].Reason} -} -func (p *Provider) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: Operations()["VerifyCredential"].Reason} -} diff --git a/services/core/internal/sandbox/docker/resources.go b/services/core/internal/sandbox/docker/resources.go index d48dfcb17..863af750a 100644 --- a/services/core/internal/sandbox/docker/resources.go +++ b/services/core/internal/sandbox/docker/resources.go @@ -13,10 +13,6 @@ import ( "github.com/moby/moby/client" ) -var _ runtimeobs.Source = (*Provider)(nil) - -func (*Provider) ObservationProviderType() string { return "docker" } - // Observe is read-only. Inspect verifies allocation ownership before Docker // statistics are requested; it never renews or changes the container. func (p *Provider) Observe(ctx context.Context, target runtimeobs.Target) (runtimeobs.Sample, error) { @@ -91,7 +87,3 @@ func sampleFromDocker(inspected container.InspectResponse, stats dockerStatsResp } return sample, nil } - -func (p *Provider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/sandbox/e2b/configuration.go b/services/core/internal/sandbox/e2b/configuration.go index 752fd60c7..5a00ed53e 100644 --- a/services/core/internal/sandbox/e2b/configuration.go +++ b/services/core/internal/sandbox/e2b/configuration.go @@ -32,7 +32,8 @@ func configuration(s sandbox.Selection) *DeploymentConfiguration { return c } func (ConfigurationAdapter) Requirements() sandbox.ConfigurationRequirements { - return sandbox.ConfigurationRequirements{Credential: sandbox.Required, PublicOrigin: sandbox.Required, Discovery: providercontract.Support{State: providercontract.Supported}} + return sandbox.ConfigurationRequirements{Credential: sandbox.Required, PublicOrigin: sandbox.Required, Discovery: providercontract.Support{State: providercontract.Supported}, + SelectionDiscovery: providercontract.Support{State: providercontract.Supported}, CredentialVerification: providercontract.Support{State: providercontract.Supported}} } func (ConfigurationAdapter) DecodeInput(public, secret json.RawMessage) (sandbox.Configuration, error) { var p publicConfiguration diff --git a/services/core/internal/sandbox/e2b/credential.go b/services/core/internal/sandbox/e2b/credential.go index 88df2e1b7..42452ff62 100644 --- a/services/core/internal/sandbox/e2b/credential.go +++ b/services/core/internal/sandbox/e2b/credential.go @@ -9,7 +9,11 @@ import ( // VerifyCredential is read-only and bounded. A public readable template alone // does not prove team ownership. References are one bounded Core-owned page. -func (p *Provider) VerifyCredential(ctx context.Context, refs []sandbox.Reference) error { +func (ConfigurationAdapter) VerifyCredential(ctx context.Context, c sandbox.DirectConfig, refs []sandbox.Reference) error { + p, err := newDirect(c) + if err != nil { + return err + } ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if len(refs) > MaxCredentialReferences { @@ -31,7 +35,7 @@ func (p *Provider) VerifyCredential(ctx context.Context, refs []sandbox.Referenc case "team_mismatch", "invalid": return sandbox.ErrCredentialOwnership case "": - if out.DeploymentValid && out.Info == nil && out.Command == nil && out.Observations == nil && out.TemplateBuild == nil { + if out.DeploymentValid && out.Info == nil && out.Command == nil && out.Observation == nil && out.TemplateBuild == nil { return nil } } diff --git a/services/core/internal/sandbox/e2b/deployment.go b/services/core/internal/sandbox/e2b/deployment.go index cc28e153f..3391d3f3e 100644 --- a/services/core/internal/sandbox/e2b/deployment.go +++ b/services/core/internal/sandbox/e2b/deployment.go @@ -70,25 +70,29 @@ func WithTemplateBuild(input sandbox.Selection, build *DeploymentBuild) sandbox. } // DiscoverSelection checks the immutable native build without allocating compute. -func (p *Provider) DiscoverSelection(ctx context.Context, s sandbox.Selection) (sandbox.Selection, error) { +func (ConfigurationAdapter) DiscoverSelection(ctx context.Context, c sandbox.DirectConfig) (sandbox.Selection, error) { + p, err := newDirect(c) + if err != nil { + return sandbox.Selection{}, err + } build, err := p.ValidateDeployment(ctx) if err != nil { if errors.Is(err, sandbox.ErrCredentialRejected) || errors.Is(err, sandbox.ErrCredentialOwnership) { - return s, err + return sandbox.Selection{}, err } if errors.Is(err, sandbox.ErrInvalid) { - return s, sandbox.ErrConfigurationSelection + return sandbox.Selection{}, sandbox.ErrConfigurationSelection } - return s, sandbox.ErrConfigurationUnconfirmed + return sandbox.Selection{}, sandbox.ErrConfigurationUnconfirmed } recorded := &DeploymentBuild{Status: build.Status, CPUs: int32(build.CPUs), MemoryMiB: int32(build.MemoryMiB)} if build.RootDiskMiB != nil && *build.RootDiskMiB <= math.MaxInt32 { disk := int32(*build.RootDiskMiB) recorded.RootDiskMiB = &disk } - s = WithTemplateBuild(s, recorded) + s := WithTemplateBuild(c.Selection, recorded) if err := ValidateSpecification(s.DeploymentSpec); err != nil { - return s, &sandbox.ValidationError{Param: "resources", Message: "E2B template build resources are outside the supported sandbox limits; select another build"} + return sandbox.Selection{}, &sandbox.ValidationError{Param: "resources", Message: "E2B template build resources are outside the supported sandbox limits; select another build"} } return s, nil } diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index 7e9a96f25..7d701cf51 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -5,7 +5,6 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) @@ -17,7 +16,6 @@ const MaxOutputBytes = 1024 * 1024 const MaxRequestBytes = 72 * 1024 * 1024 const MaxResponseBytes = 16 * 1024 * 1024 const MaxCredentialReferences = 32 -const MaxObservationReferences = runtimeobs.MaxBatchTargets const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. @@ -40,17 +38,6 @@ func (q Request) Validate() error { return sandbox.ErrInvalid } switch q.Operation { - case "observe": - if len(q.References) < 1 || len(q.References) > MaxObservationReferences { - return sandbox.ErrInvalid - } - seen := map[sandbox.Reference]bool{} - for _, r := range q.References { - if !validReference(r) || seen[r] { - return sandbox.ErrInvalid - } - seen[r] = true - } case "verify_credential": if len(q.References) > MaxCredentialReferences { return sandbox.ErrInvalid @@ -74,7 +61,7 @@ type Request struct { Operation string Config Config Reference sandbox.Reference - // References lists the allocations of one read-only observe request. + // References lists the allocations of one verify_credential request. References []sandbox.Reference `json:",omitempty"` Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` @@ -90,7 +77,7 @@ type Response struct { TemplateBuild *TemplateBuild `json:",omitempty"` Templates []TemplateSummary `json:",omitempty"` Builds []ReadyBuild `json:",omitempty"` - Observations []Observation `json:",omitempty"` + Observation *Observation `json:",omitempty"` } func (r Response) Validate() error { diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 6d79d9544..023dc781e 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -83,8 +83,7 @@ func main() { values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, - "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, - "MAX_OBSERVATION_REFERENCES": e2b.MaxObservationReferences, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, + "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), "REFERENCE_FIELDS": fields(reflect.TypeFor[sandbox.Reference]()), @@ -131,7 +130,7 @@ func fixtures() []byte { for _, operation := range e2b.HelperOperations() { copy := q copy.Operation = operation - if operation == "observe" || operation == "verify_credential" { + if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } add("request", operation, true, copy) @@ -148,32 +147,22 @@ func fixtures() []byte { value[item.field] = item.value add("request", item.name, false, value) } - for _, operation := range []string{"observe", "verify_credential"} { - limit := e2b.MaxObservationReferences - if operation == "verify_credential" { - limit = e2b.MaxCredentialReferences - } - for _, count := range []int{0, limit, limit + 1} { - copy := q - copy.Operation = operation - copy.References = make([]sandbox.Reference, count) - for index := range copy.References { - copy.References[index] = r - copy.References[index].AllocationID = fmt.Sprintf("%08x-3333-4333-8333-333333333333", index+1) - } - add("request", fmt.Sprintf("%s-count-%d", operation, count), count <= limit && (operation != "observe" || count > 0), copy) - } - for _, refs := range []any{map[string]any{}, "invalid", []any{nil}} { - value := object(q) - value["Operation"] = operation - value["References"] = refs - add("request", operation+"-references-type", false, value) + for _, count := range []int{0, e2b.MaxCredentialReferences, e2b.MaxCredentialReferences + 1} { + copy := q + copy.Operation = "verify_credential" + copy.References = make([]sandbox.Reference, count) + for index := range copy.References { + copy.References[index] = r + copy.References[index].AllocationID = fmt.Sprintf("%08x-3333-4333-8333-333333333333", index+1) } + add("request", fmt.Sprintf("verify_credential-count-%d", count), count <= e2b.MaxCredentialReferences, copy) + } + for _, refs := range []any{map[string]any{}, "invalid", []any{nil}} { + value := object(q) + value["Operation"] = "verify_credential" + value["References"] = refs + add("request", "verify_credential-references-type", false, value) } - duplicate := q - duplicate.Operation = "observe" - duplicate.References = []sandbox.Reference{r, r} - add("request", "duplicate-observation", false, duplicate) for _, code := range e2b.HelperErrors() { add("response", "error-"+code, true, e2b.Response{Version: e2b.ProtocolVersion, ErrorCode: code}) } diff --git a/services/core/internal/sandbox/e2b/observations.go b/services/core/internal/sandbox/e2b/observations.go index e931805ad..063ba1307 100644 --- a/services/core/internal/sandbox/e2b/observations.go +++ b/services/core/internal/sandbox/e2b/observations.go @@ -9,21 +9,16 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var ( - _ runtimeobs.Source = (*Provider)(nil) - _ runtimeobs.BatchSource = (*Provider)(nil) -) - // maxClockLead tolerates an E2B metrics timestamp slightly ahead of Core's // clock. A larger lead is treated as an unavailable sample, never as fresh data. const maxClockLead = 30 * time.Second -// Observation is one allocation's latest E2B metrics point. Status is -// observed, not_running, unavailable or ownership; only observed carries -// values. A malformed E2B point is unavailable for its row only. Values keep E2B's units: CPUUsedPct is a percentage of all -// CPUCount cores, and memory and disk are in bytes. +// Observation is the requested allocation's latest E2B metrics point. Status +// is observed, not_running, unavailable or ownership; only observed carries +// values. A malformed E2B point is unavailable. Values keep E2B's units: +// CPUUsedPct is a percentage of all CPUCount cores, and memory and disk are in +// bytes. type Observation struct { - sandbox.Reference Status string ObservedAt, StartedAt *time.Time `json:",omitempty"` CPUCount, CPUUsedPct *float64 `json:",omitempty"` @@ -31,83 +26,46 @@ type Observation struct { DiskUsed, DiskTotal *uint64 `json:",omitempty"` } -func (*Provider) ObservationProviderType() string { return "e2b" } - -// Observe reads one allocation through the same helper request as ObserveBatch. +// Observe reads one allocation with one helper request. The helper takes the +// sandbox ID from its private receipt, confirms the running sandbox by its +// allocation labels and reads E2B's metrics. It never connects to, renews or +// changes a sandbox. func (p *Provider) Observe(ctx context.Context, target runtimeobs.Target) (runtimeobs.Sample, error) { - results, _ := p.ObserveBatch(ctx, []runtimeobs.Target{target}) - return results[0].Sample, results[0].Err -} - -// ObserveBatch reads up to runtimeobs.MaxBatchTargets allocations with one -// helper request. The helper takes sandbox IDs from its private receipts, -// confirms each running sandbox by its allocation labels and reads E2B's batch -// metrics once. It never connects to, renews or changes a sandbox. -func (p *Provider) ObserveBatch(ctx context.Context, targets []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - results := make([]runtimeobs.BatchResult, len(targets)) - references := make([]sandbox.Reference, 0, len(targets)) - positions := make([]int, 0, len(targets)) - for index, target := range targets { - reference := sandbox.Reference{TenantID: target.TenantID, EnvironmentID: target.EnvironmentID, AllocationID: target.Instance.AllocationID} - switch { - case target.Mode != runtimeobs.ModeManaged || !validReference(reference) || len(targets) > runtimeobs.MaxBatchTargets: - results[index].Err = sandbox.ErrInvalid - case target.Instance.ProviderKey != p.config.InstallationID: - results[index].Err = sandbox.ErrOwnership - default: - references = append(references, reference) - positions = append(positions, index) - } - } - if len(references) == 0 { - return results, nil + reference := sandbox.Reference{TenantID: target.TenantID, EnvironmentID: target.EnvironmentID, AllocationID: target.Instance.AllocationID} + if target.Mode != runtimeobs.ModeManaged || !validReference(reference) { + return runtimeobs.Sample{}, sandbox.ErrInvalid } - observations, err := p.observe(ctx, references) - now := p.now() - for offset, index := range positions { - if err != nil { - results[index].Err = err - continue - } - results[index].Sample, results[index].Err = sampleFromObservation(observations[offset], now) + if target.Instance.ProviderKey != p.config.InstallationID { + return runtimeobs.Sample{}, sandbox.ErrOwnership } - return results, nil -} - -func (p *Provider) observe(ctx context.Context, references []sandbox.Reference) ([]Observation, error) { deadline, ok := ctx.Deadline() if !ok { - return nil, sandbox.ErrInvalid + return runtimeobs.Sample{}, sandbox.ErrInvalid } if err := ctx.Err(); err != nil { - return nil, err + return runtimeobs.Sample{}, err } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: "observe", Config: p.config, References: references, Deadline: deadline}) + out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: "observe", Config: p.config, Reference: reference, Deadline: deadline}) if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr + return runtimeobs.Sample{}, ctxErr } if err != nil || out.Version != ProtocolVersion || out.Info != nil || out.Command != nil || out.DeploymentValid || out.TemplateBuild != nil { - return nil, runtimeobs.ErrUnavailable + return runtimeobs.Sample{}, runtimeobs.ErrUnavailable } switch out.ErrorCode { case "": case "invalid": - return nil, sandbox.ErrInvalid + return runtimeobs.Sample{}, sandbox.ErrInvalid case "ownership": - return nil, sandbox.ErrOwnership + return runtimeobs.Sample{}, sandbox.ErrOwnership default: - // E2B API failures, including a rejected credential, leave rows unavailable. - return nil, runtimeobs.ErrUnavailable - } - if len(out.Observations) != len(references) { - return nil, sandbox.ErrInvalid + // E2B API failures, including a rejected credential, are unavailable. + return runtimeobs.Sample{}, runtimeobs.ErrUnavailable } - for index, observation := range out.Observations { - if observation.Reference != references[index] { - return nil, sandbox.ErrOwnership - } + if out.Observation == nil { + return runtimeobs.Sample{}, sandbox.ErrInvalid } - return out.Observations, nil + return sampleFromObservation(*out.Observation, p.now()) } // sampleFromObservation maps E2B's latest point to the provider-neutral @@ -125,7 +83,7 @@ func sampleFromObservation(observation Observation, now time.Time) (runtimeobs.S // An unknown status breaks Core's own helper protocol. return runtimeobs.Sample{}, sandbox.ErrInvalid } - // Malformed provider data leaves this row unavailable, not the whole page. + // Malformed provider data leaves this sample unavailable. if observation.ObservedAt == nil || observation.StartedAt == nil || observation.CPUCount == nil || observation.CPUUsedPct == nil || observation.MemUsed == nil || observation.MemTotal == nil || !finite(*observation.CPUCount) || *observation.CPUCount <= 0 || !finite(*observation.CPUUsedPct) || *observation.CPUUsedPct < 0 || *observation.MemTotal == 0 { @@ -160,7 +118,3 @@ func sampleFromObservation(observation Observation, now time.Time) (runtimeobs.S } func finite(value float64) bool { return !math.IsNaN(value) && !math.IsInf(value, 0) } - -func (p *Provider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/sandbox/e2b/observations_test.go b/services/core/internal/sandbox/e2b/observations_test.go index c736e862c..04acefd4d 100644 --- a/services/core/internal/sandbox/e2b/observations_test.go +++ b/services/core/internal/sandbox/e2b/observations_test.go @@ -6,56 +6,48 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/google/uuid" ) -func TestObserveBatchMapsMetricsAndKeepsUnmeasuredValuesNull(t *testing.T) { +func TestObserveMapsMetricsAndKeepsUnmeasuredValuesNull(t *testing.T) { p, caller, running := fixture(t) - stopped := sandbox.Reference{TenantID: uuid.NewString(), EnvironmentID: uuid.NewString(), AllocationID: uuid.NewString()} now := time.Date(2026, 9, 25, 10, 31, 37, 0, time.UTC) p.now = func() time.Time { return now } started, observed := now.Add(-3*time.Second), now.Add(time.Second) // E2B's clock leads by one second. count, percent, memoryUsed, memoryTotal, diskUsed, diskTotal := 2.0, 19.55, uint64(183836672), uint64(2079141888), uint64(1593188352), uint64(23511863296) - caller.response.Observations = []Observation{ - {Reference: running, Status: "observed", ObservedAt: &observed, StartedAt: &started, CPUCount: &count, CPUUsedPct: &percent, - MemUsed: &memoryUsed, MemTotal: &memoryTotal, DiskUsed: &diskUsed, DiskTotal: &diskTotal}, - {Reference: stopped, Status: "not_running"}, - } - targets := []runtimeobs.Target{} - for _, reference := range []sandbox.Reference{running, stopped} { - targets = append(targets, runtimeobs.Target{TenantID: reference.TenantID, EnvironmentID: reference.EnvironmentID, Mode: runtimeobs.ModeManaged, - Instance: runtimeobs.Instance{AllocationID: reference.AllocationID, ProviderKey: p.config.InstallationID}}) - } - results, err := p.ObserveBatch(bounded(t), targets) - if err != nil || len(caller.requests) != 1 || caller.requests[0].Operation != "observe" || len(caller.requests[0].References) != 2 || - caller.requests[0].References[1] != stopped || caller.requests[0].Deadline.IsZero() { - t.Fatalf("batch was not one bounded helper request: err=%v %+v", err, caller.requests) - } - sample := results[0].Sample - if results[0].Err != nil || !sample.ObservedAt.Equal(now) || !sample.StartedAt.Equal(started) || + caller.response.Observation = &Observation{Status: "observed", ObservedAt: &observed, StartedAt: &started, CPUCount: &count, CPUUsedPct: &percent, + MemUsed: &memoryUsed, MemTotal: &memoryTotal, DiskUsed: &diskUsed, DiskTotal: &diskTotal} + target := runtimeobs.Target{TenantID: running.TenantID, EnvironmentID: running.EnvironmentID, Mode: runtimeobs.ModeManaged, + Instance: runtimeobs.Instance{AllocationID: running.AllocationID, ProviderKey: p.config.InstallationID}} + sample, err := p.Observe(bounded(t), target) + if err != nil || len(caller.requests) != 1 || caller.requests[0].Operation != "observe" || + caller.requests[0].Reference != running || caller.requests[0].Deadline.IsZero() { + t.Fatalf("observation was not one bounded helper request: err=%v %+v", err, caller.requests) + } + if !sample.ObservedAt.Equal(now) || !sample.StartedAt.Equal(started) || *sample.CPUUtilizationRatio != .1955 || *sample.CPUCapacityCores != 2 || sample.CPUUsageSecondsTotal != nil || *sample.MemoryUsageBytes != memoryUsed || *sample.MemoryLimitBytes != memoryTotal || *sample.DiskUsageBytes != diskUsed || *sample.DiskLimitBytes != diskTotal { - t.Fatalf("metrics were not mapped: %+v %v", sample, results[0].Err) - } - if !errors.Is(results[1].Err, runtimeobs.ErrNotRunning) { - t.Fatalf("absent sandbox = %v", results[1].Err) + t.Fatalf("metrics were not mapped: %+v", sample) } - caller.response.Observations[0].DiskTotal = nil - results, _ = p.ObserveBatch(bounded(t), targets) - if results[0].Err != nil || results[0].Sample.DiskUsageBytes != nil || results[0].Sample.DiskLimitBytes != nil { - t.Fatalf("unreported disk was not null: %+v %v", results[0].Sample, results[0].Err) + caller.response.Observation.DiskTotal = nil + if sample, err = p.Observe(bounded(t), target); err != nil || sample.DiskUsageBytes != nil || sample.DiskLimitBytes != nil { + t.Fatalf("unreported disk was not null: %+v %v", sample, err) + } + caller.response.Observation.MemTotal = nil + if _, err = p.Observe(bounded(t), target); !errors.Is(err, runtimeobs.ErrUnavailable) { + t.Fatalf("malformed point = %v", err) + } + caller.response.Observation = &Observation{Status: "not_running"} + if _, err = p.Observe(bounded(t), target); !errors.Is(err, runtimeobs.ErrNotRunning) { + t.Fatalf("absent sandbox = %v", err) } - caller.response.Observations[0].MemTotal = nil - results, _ = p.ObserveBatch(bounded(t), targets) - if !errors.Is(results[0].Err, runtimeobs.ErrUnavailable) || !errors.Is(results[1].Err, runtimeobs.ErrNotRunning) { - t.Fatalf("a malformed point affected more than its row: %+v", results) + caller.response.Observation = nil + if _, err = p.Observe(bounded(t), target); err == nil { + t.Fatal("a success without an observation was accepted") } caller.response.ErrorCode = "unconfirmed" - results, _ = p.ObserveBatch(bounded(t), targets) - if !errors.Is(results[0].Err, runtimeobs.ErrUnavailable) || !errors.Is(results[1].Err, runtimeobs.ErrUnavailable) { - t.Fatalf("E2B failure was not unavailable: %+v", results) + if _, err = p.Observe(bounded(t), target); !errors.Is(err, runtimeobs.ErrUnavailable) { + t.Fatalf("E2B failure was not unavailable: %v", err) } } diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index 14530ebf9..a3da4688e 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -3,38 +3,27 @@ package e2b import ( "context" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ sandbox.CheckpointProvider = (*Provider)(nil) -var _ sandbox.SelectionDiscoverer = (*Provider)(nil) -var _ sandbox.CredentialVerifier = (*Provider)(nil) -var _ runtimeobs.BatchSource = (*Provider)(nil) - // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Supported}, - "ObserveBatch": {State: providercontract.Supported}, - "DiscoverSelection": {State: providercontract.Supported}, - "VerifyCredential": {State: providercontract.Supported}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index 3328eb0db..c07924250 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -56,7 +56,7 @@ func Discover(ctx context.Context, caller Caller, binary, apiKey, apiURL, domain if out.ErrorCode == "invalid" { return Response{}, sandbox.ErrInvalid } - if out.ErrorCode != "" || out.Info != nil || out.Command != nil || out.TemplateBuild != nil || out.Observations != nil { + if out.ErrorCode != "" || out.Info != nil || out.Command != nil || out.TemplateBuild != nil || out.Observation != nil { return Response{}, sandbox.ErrComputeUnconfirmed } if operation == "list_templates" { @@ -151,6 +151,33 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { } return &Provider{config: c, caller: caller, now: time.Now}, nil } + +// BuildDirect builds the Provider for one direct-mode selection. +func BuildDirect(c sandbox.DirectConfig) (sandbox.SandboxProvider, error) { return newDirect(c) } + +func newDirect(c sandbox.DirectConfig) (*Provider, error) { + deployment := configuration(c.Selection) + if deployment == nil { + return nil, errors.New("E2B deployment configuration is unavailable") + } + binary, state, err := InstalledPaths(c.ProcessPaths) + if err != nil { + return nil, err + } + // Only a candidate that omitted its resources has none; its validation + // reads them from the template build before the candidate is rebuilt. + var resources *sandbox.Resources + if c.Selection.DeploymentSpec.Resources != (sandbox.Resources{}) { + resources = &c.Selection.DeploymentSpec.Resources + } + provider, err := NewWithCaller(Config{Binary: binary, StateDir: state, + Resources: resources, InstallationID: c.InstallationID, APIKey: deployment.APIKey, Template: deployment.Template, + APIURL: deployment.APIURL, Domain: deployment.Domain, TimeoutSeconds: 3600}, &ProcessCaller{Fence: c.Fence}) + if err != nil { + return nil, errors.New("E2B provider cannot load; check the installed helper and private state directory") + } + return provider, nil +} func (p *Provider) call(ctx context.Context, operation string, r sandbox.Reference, b *sandbox.Bootstrap, command *sandbox.Command) (Response, error) { deadline, ok := ctx.Deadline() if !ok || (operation != "validate_deployment" && !validReference(r)) { @@ -216,7 +243,7 @@ func (p *Provider) ValidateDeployment(ctx context.Context) (TemplateBuild, error return TemplateBuild{}, err } build := out.TemplateBuild - if !out.DeploymentValid || out.Info != nil || out.Command != nil || out.Observations != nil || build == nil || build.Status != "ready" || + if !out.DeploymentValid || out.Info != nil || out.Command != nil || out.Observation != nil || build == nil || build.Status != "ready" || build.CPUs == 0 || build.MemoryMiB == 0 || build.RootDiskMiB != nil && *build.RootDiskMiB == 0 || p.config.Resources != nil && (build.CPUs != p.config.Resources.CPUs || build.MemoryMiB != p.config.Resources.MemoryMiB) { return TemplateBuild{}, sandbox.ErrComputeUnconfirmed diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index f59c9ed02..13e81ff96 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -1,49 +1,25 @@ package microsandbox -import ( - "context" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" -) - -var _ sandbox.CheckpointProvider = (*Provider)(nil) -var _ sandbox.SelectionDiscoverer = (*Provider)(nil) -var _ sandbox.CredentialVerifier = (*Provider)(nil) -var _ runtimeobs.BatchSource = (*Provider)(nil) +import "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" // Operations is this adapter's complete authored resource contract. func Operations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Supported}, - "NewCompute": {State: providercontract.Supported}, - "GetCompute": {State: providercontract.Supported}, - "Suspend": {State: providercontract.Supported}, - "Resume": {State: providercontract.Supported}, - "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, - "RunCommandCompute": {State: providercontract.Supported}, - "ResumeCompute": {State: providercontract.Supported}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Supported}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "microsandbox_does_not_support_batch_observation"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "microsandbox_does_not_support_selection_discovery"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "microsandbox_does_not_support_credential_verification"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteSnapshot": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, + "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: Operations()["ObserveBatch"].Reason} -} -func (p *Provider) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: Operations()["DiscoverSelection"].Reason} -} -func (p *Provider) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: Operations()["VerifyCredential"].Reason} -} diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 6ba4eeafc..d876702be 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -16,7 +16,6 @@ type Provider struct { } var _ sandbox.SandboxProvider = (*Provider)(nil) -var _ sandbox.CheckpointProvider = (*Provider)(nil) func NewWithCaller(c Config, caller Caller) (*Provider, error) { if c.Validate() != nil || caller == nil { @@ -213,9 +212,3 @@ func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generati } return c, nil } - -// Quiescent includes a helper that outlived the caller's canceled context. -func (p *Provider) Quiescent() bool { - v, ok := p.caller.(interface{ Quiescent() bool }) - return ok && v.Quiescent() -} diff --git a/services/core/internal/sandbox/microsandbox/resources.go b/services/core/internal/sandbox/microsandbox/resources.go index 6b4619147..ce586b389 100644 --- a/services/core/internal/sandbox/microsandbox/resources.go +++ b/services/core/internal/sandbox/microsandbox/resources.go @@ -10,10 +10,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ runtimeobs.Source = (*Provider)(nil) - -func (*Provider) ObservationProviderType() string { return "microsandbox" } - // Observe reads one point-in-time native metrics snapshot through the existing // one-shot helper. The persisted compute receipt selects the exact generation; // browser input and provider display names never select a sandbox. @@ -74,7 +70,3 @@ func sampleFromMetrics(config Config, metrics Metrics) (runtimeobs.Sample, error MemoryUsageBytes: &memoryUsage, MemoryLimitBytes: &memoryLimit, }, nil } - -func (p *Provider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/sandbox/node/connection_test.go b/services/core/internal/sandbox/node/connection_test.go index 30f88ecd8..abee501b3 100644 --- a/services/core/internal/sandbox/node/connection_test.go +++ b/services/core/internal/sandbox/node/connection_test.go @@ -176,7 +176,7 @@ func TestCopiedIdentityCannotReplaceNodeWithInflightCreate(t *testing.T) { hub.mu.Unlock() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() created := make(chan error, 1) go func() { _, err := proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); created <- err }() diff --git a/services/core/internal/sandbox/node/creation_settlement_test.go b/services/core/internal/sandbox/node/creation_settlement_test.go index 02ef95248..09d8daec5 100644 --- a/services/core/internal/sandbox/node/creation_settlement_test.go +++ b/services/core/internal/sandbox/node/creation_settlement_test.go @@ -73,7 +73,7 @@ func TestNodeCarriesCreationSettlementWithoutConvertingFailureToSuccess(t *testi } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) for _, operation := range []func(context.Context) (sandbox.Info, error){ func(ctx context.Context) (sandbox.Info, error) { return proxy.Create(ctx, sandbox.Bootstrap{Reference: ref}) diff --git a/services/core/internal/sandbox/node/docker_live_test.go b/services/core/internal/sandbox/node/docker_live_test.go index 13ab16b51..48695bfff 100644 --- a/services/core/internal/sandbox/node/docker_live_test.go +++ b/services/core/internal/sandbox/node/docker_live_test.go @@ -76,7 +76,7 @@ func TestDockerNodeTransportLifecycle(t *testing.T) { } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() defer func() { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) diff --git a/services/core/internal/sandbox/node/generation_connection_test.go b/services/core/internal/sandbox/node/generation_connection_test.go index d39b68459..1c3e59d6c 100644 --- a/services/core/internal/sandbox/node/generation_connection_test.go +++ b/services/core/internal/sandbox/node/generation_connection_test.go @@ -63,7 +63,7 @@ func TestGenerationWireRoutesOldOwnershipAndCurrentTargetSeparately(t *testing.T }() wait(t, func() bool { return hub.Online(id.NodeID) }) for _, generation := range []uint64{1, 17, 9} { - proxy := hub.GenerationProvider("docker", docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { return id.NodeID, generation, nil }) + proxy := hub.GenerationProvider(docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { return id.NodeID, generation, nil }) ref := reference() if _, err := proxy.GetInfo(ctx, ref); err != nil { t.Fatal("retained generation info failed", generation, err) diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 95613b674..047bbe4e1 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -15,7 +15,10 @@ type GenerationProvider struct { SpecificationDigest string Provider sandbox.SandboxProvider Probe func(context.Context) error - Close func() + // Quiescent reports that no helper outlived its canceled caller; nil + // means always quiescent. + Quiescent func() bool + Close func() } type GenerationManagerOptions struct { @@ -217,11 +220,7 @@ func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRe m.mu.Unlock() return sandbox.ErrOwnership } - quiet := true - if provider, ok := g.value.Provider.(interface{ Quiescent() bool }); ok { - quiet = provider.Quiescent() - } - if !quiet || g.refs != 0 || g.removing || m.target.Generation == grant.Generation || m.target.ServingGeneration != nil && *m.target.ServingGeneration == grant.Generation { + if g.value.Quiescent != nil && !g.value.Quiescent() || g.refs != 0 || g.removing || m.target.Generation == grant.Generation || m.target.ServingGeneration != nil && *m.target.ServingGeneration == grant.Generation { m.mu.Unlock() return ErrUnavailable } @@ -265,7 +264,7 @@ func (m *GenerationManager) prepareLoop() { if candidate == nil || candidate.removing || candidate.collecting || candidate.preparing || candidate.refs != 0 || candidate.value.Provider != nil && !candidate.repairing || time.Now().Before(candidate.retryAt) { continue } - if provider, ok := candidate.value.Provider.(interface{ Quiescent() bool }); ok && !provider.Quiescent() { + if candidate.value.Quiescent != nil && !candidate.value.Quiescent() { continue } g = candidate diff --git a/services/core/internal/sandbox/node/generations_test.go b/services/core/internal/sandbox/node/generations_test.go index 9292bbbb2..8831d5321 100644 --- a/services/core/internal/sandbox/node/generations_test.go +++ b/services/core/internal/sandbox/node/generations_test.go @@ -116,13 +116,6 @@ func TestRetentionReplyMustMatchWholePendingExchange(t *testing.T) { } } -type helperOwnedProvider struct { - *fakeProvider - caller *microsandbox.ProcessCaller -} - -func (p *helperOwnedProvider) Quiescent() bool { return p.caller.Quiescent() } - func TestGrantedDropWaitsForReferencesAndActualHelperExit(t *testing.T) { root := t.TempDir() helper := filepath.Join(root, "helper") @@ -142,7 +135,7 @@ func TestGrantedDropWaitsForReferencesAndActualHelperExit(t *testing.T) { defer func() { _ = os.WriteFile(release, nil, 0600); wait(t, caller.Quiescent) }() m := generationFixture(3) m.target.ServingGeneration = nil - m.values[1].value.Provider = &helperOwnedProvider{fakeProvider: &fakeProvider{}, caller: caller} + m.values[1].value.Provider, m.values[1].value.Quiescent = &fakeProvider{}, caller.Quiescent m.options.Remove = func(context.Context, GenerationProvider) error { return os.Remove(artifact) } _, _, unref, err := m.Acquire(1) if err != nil { diff --git a/services/core/internal/sandbox/node/hub_lifetime_test.go b/services/core/internal/sandbox/node/hub_lifetime_test.go index ed5d0d379..37b07c74b 100644 --- a/services/core/internal/sandbox/node/hub_lifetime_test.go +++ b/services/core/internal/sandbox/node/hub_lifetime_test.go @@ -46,7 +46,7 @@ func assertInfoResponsive(t *testing.T, hub *Hub, id Identity) { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - info, err := hub.Proxy(id.NodeID, id.Provider, docker.Operations(), 1).GetInfo(ctx, reference()) + info, err := hub.Proxy(id.NodeID, docker.Operations(), 1).GetInfo(ctx, reference()) if err != nil || info.ProviderID != "retained" { t.Fatalf("unrelated node RPC blocked: info=%+v err=%v", info, err) } @@ -287,7 +287,7 @@ func TestHubSendQueueRespectsCallerCancellation(t *testing.T) { defer cancel() done := make(chan error, 1) go func() { - _, err := hub.Proxy(id.NodeID, id.Provider, docker.Operations(), 1).GetInfo(ctx, reference()) + _, err := hub.Proxy(id.NodeID, docker.Operations(), 1).GetInfo(ctx, reference()) done <- err }() select { diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index aab23036f..9aca8cd5a 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -125,7 +125,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing t.Fatal("running node did not retain lifetime identity lock") } r := reference() - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) createCtx, stopCreate := context.WithTimeout(ctx, 150*time.Millisecond) defer stopCreate() createDone := make(chan error, 1) @@ -168,7 +168,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := NewHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) - p := h.Proxy(uuid.NewString(), "docker", docker.Operations(), 1) + p := h.Proxy(uuid.NewString(), docker.Operations(), 1) if sandbox.SupportsCheckpoint(p) { t.Fatal("docker advertised checkpoint") } diff --git a/services/core/internal/sandbox/node/observations.go b/services/core/internal/sandbox/node/observations.go index 515606640..7129d2f33 100644 --- a/services/core/internal/sandbox/node/observations.go +++ b/services/core/internal/sandbox/node/observations.go @@ -3,16 +3,11 @@ package node import ( "context" "errors" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -var _ runtimeobs.Source = (*provider)(nil) - -func (p *provider) ObservationProviderType() string { return p.kind } - func observationReference(target runtimeobs.Target) sandbox.Reference { return sandbox.Reference{TenantID: target.TenantID, EnvironmentID: target.EnvironmentID, AllocationID: target.Instance.AllocationID} } @@ -39,18 +34,3 @@ func (p *provider) Observe(ctx context.Context, target runtimeobs.Target) (runti } return *out.Sample, nil } - -func observeProvider(ctx context.Context, provider sandbox.SandboxProvider, target runtimeobs.Target) (runtimeobs.Sample, error) { - if err := providercontract.Require(provider, "Observe"); err != nil { - return runtimeobs.Sample{}, err - } - source, ok := provider.(runtimeobs.Source) - if !ok { - return runtimeobs.Sample{}, providercontract.ErrContract - } - return source.Observe(ctx, target) -} - -func (p *provider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/sandbox/node/observations_test.go b/services/core/internal/sandbox/node/observations_test.go index f9b7e1656..23a5cbfdb 100644 --- a/services/core/internal/sandbox/node/observations_test.go +++ b/services/core/internal/sandbox/node/observations_test.go @@ -89,15 +89,12 @@ func TestObservationsRouteThroughAssignedNodeWithoutLifecycleCalls(t *testing.T) stopSecond := runObservationNode(t, hub, server.URL, second, b) ra, rb := reference(), reference() assignments := map[sandbox.Reference]string{ra: first.NodeID, rb: second.NodeID} - source := hub.GenerationProvider("docker", docker.Operations(), func(_ context.Context, r sandbox.Reference) (string, uint64, error) { + source := hub.GenerationProvider(docker.Operations(), func(_ context.Context, r sandbox.Reference) (string, uint64, error) { if id, ok := assignments[r]; ok { return id, 1, nil } return "", 0, sandbox.ErrOwnership - }).(runtimeobs.Source) - if typed := source.ObservationProviderType(); typed != "docker" { - t.Fatal("provider type lost", typed) - } + }) for _, test := range []struct { ref sandbox.Reference provider *observationProvider diff --git a/services/core/internal/sandbox/node/operations_test.go b/services/core/internal/sandbox/node/operations_test.go index 2cd469fda..e508988b5 100644 --- a/services/core/internal/sandbox/node/operations_test.go +++ b/services/core/internal/sandbox/node/operations_test.go @@ -37,7 +37,7 @@ func TestUnsupportedWireIsExplicitAndDoesNotInvokeProvider(t *testing.T) { } } func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { - p := (&Hub{}).GenerationProvider("docker", docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { + p := (&Hub{}).GenerationProvider(docker.Operations(), func(context.Context, sandbox.Reference) (string, uint64, error) { t.Fatal("unsupported call resolved a node") return "", 0, nil }).(*provider) @@ -47,9 +47,6 @@ func TestUnsupportedProxyRejectsBeforeNodeResolution(t *testing.T) { if _, err := p.Initial(t.Context(), reference()); !errors.Is(err, providercontract.ErrUnsupported) { t.Fatal(err) } - if _, err := p.ObserveBatch(t.Context(), nil); !errors.Is(err, providercontract.ErrUnsupported) { - t.Fatal(err) - } } func TestNodeOperationMappingCoversForwardedMethods(t *testing.T) { for _, method := range []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand", "Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "Observe"} { diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 2569a6632..3f892e9bb 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -10,26 +10,21 @@ import ( func (*fakeProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } func (*fakeProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { @@ -62,46 +57,22 @@ func (*fakeProvider) ResumeCompute(context.Context, sandbox.Reference, sandbox.C func (*fakeProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*fakeProvider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_operation_not_supported"} -} -func (*fakeProvider) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "fixture_operation_not_supported"} -} -func (*fakeProvider) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "fixture_operation_not_supported"} -} func (*observationProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Supported}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Observe": {State: providercontract.Supported}, } } - -func (*fakeProvider) ObservationProviderType() string { return "fixture" } -func (p *fakeProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} - -func (*observationProvider) ObservationProviderType() string { return "fixture" } -func (p *observationProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index 01daa2aa1..fc126ebe8 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -6,23 +6,20 @@ import ( "maps" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) type provider struct { hub *Hub resolveGeneration func(context.Context, sandbox.Reference) (string, uint64, error) - kind string operations providercontract.Operations } var _ sandbox.SandboxProvider = (*provider)(nil) -var _ sandbox.CheckpointProvider = (*provider)(nil) // Proxy binds a fixed node and deployment generation explicitly. -func (h *Hub) Proxy(id, kind string, declared providercontract.Operations, generation uint64) sandbox.SandboxProvider { - return h.GenerationProvider(kind, declared, func(context.Context, sandbox.Reference) (string, uint64, error) { return id, generation, nil }) +func (h *Hub) Proxy(id string, declared providercontract.Operations, generation uint64) sandbox.SandboxProvider { + return h.GenerationProvider(declared, func(context.Context, sandbox.Reference) (string, uint64, error) { return id, generation, nil }) } func (p *provider) call(ctx context.Context, q request) (response, error) { if err := providercontract.Require(p, operationMethod(q.Operation)); err != nil { @@ -72,15 +69,6 @@ func creationSettled(info *sandbox.Info, ref sandbox.Reference) bool { func (p *provider) ProviderOperations() providercontract.Operations { return maps.Clone(p.operations) } -func (*provider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "node_transport_has_no_batch_observation"} -} -func (p *provider) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "node_configuration_is_core_owned"} -} -func (p *provider) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "node_credentials_are_transport_owned"} -} func (p *provider) Create(ctx context.Context, b sandbox.Bootstrap) (sandbox.Info, error) { return p.info(ctx, request{Operation: "create", Reference: b.Reference, Bootstrap: &b}) } @@ -163,13 +151,7 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san // GenerationProvider routes every operation with allocation-owned generation, // distinct from the request's compute generation. declared is the kind's -// registered operations; the transport replaces the ones it owns. -func (h *Hub) GenerationProvider(kind string, declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { - operations := maps.Clone(declared) - if operations != nil { - operations["DiscoverSelection"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_is_core_owned"} - operations["VerifyCredential"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_credentials_are_transport_owned"} - operations["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "node_transport_has_no_batch_observation"} - } - return &provider{hub: h, kind: kind, operations: operations, resolveGeneration: resolve} +// registered operations. +func (h *Hub) GenerationProvider(declared providercontract.Operations, resolve func(context.Context, sandbox.Reference) (string, uint64, error)) sandbox.SandboxProvider { + return &provider{hub: h, operations: maps.Clone(declared), resolveGeneration: resolve} } diff --git a/services/core/internal/sandbox/node/recovery_test.go b/services/core/internal/sandbox/node/recovery_test.go index d150b6422..78d106da3 100644 --- a/services/core/internal/sandbox/node/recovery_test.go +++ b/services/core/internal/sandbox/node/recovery_test.go @@ -57,10 +57,10 @@ func TestCoreRestartFencesOldConnectionAndNodeRestartKeepsIdentity(t *testing.T) if first.Online(id.NodeID) { t.Fatal("old owner remained online") } - if _, err = first.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(context.Background(), reference()); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { + if _, err = first.Proxy(id.NodeID, docker.Operations(), 1).GetInfo(context.Background(), reference()); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("old owner request = %v", err) } - if _, err = second.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(context.Background(), reference()); err != nil { + if _, err = second.Proxy(id.NodeID, docker.Operations(), 1).GetInfo(context.Background(), reference()); err != nil { t.Fatal(err) } stop() @@ -156,7 +156,7 @@ func TestHeartbeatAcknowledgementKeepsIdleConnectionAlive(t *testing.T) { if !hub.Online(id.NodeID) { t.Fatal("idle node disconnected") } - if _, err = hub.Proxy(id.NodeID, "docker", docker.Operations(), 1).GetInfo(ctx, reference()); err != nil { + if _, err = hub.Proxy(id.NodeID, docker.Operations(), 1).GetInfo(ctx, reference()); err != nil { t.Fatal(err) } } @@ -234,7 +234,7 @@ func TestDegradedNodeRetainsObservationAndCleanup(t *testing.T) { case <-time.After(time.Second): t.Fatal("missing health") } - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() if _, err = proxy.Create(ctx, sandbox.Bootstrap{Reference: r}); !errors.Is(err, sandbox.ErrComputeUnconfirmed) { t.Fatalf("create = %v", err) diff --git a/services/core/internal/sandbox/node/timeout_test.go b/services/core/internal/sandbox/node/timeout_test.go index c849b853d..15f8da3c5 100644 --- a/services/core/internal/sandbox/node/timeout_test.go +++ b/services/core/internal/sandbox/node/timeout_test.go @@ -103,7 +103,7 @@ func TestQueuedMutationExpiresWithoutExecution(t *testing.T) { } }() wait(t, func() bool { return hub.Online(id.NodeID) }) - proxy := hub.Proxy(id.NodeID, "docker", docker.Operations(), 1) + proxy := hub.Proxy(id.NodeID, docker.Operations(), 1) r := reference() createCtx, stopCreate := context.WithTimeout(ctx, 3*time.Second) defer stopCreate() diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index 0194e3c82..03a412951 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -307,7 +307,7 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response switch q.Operation { case "observe": var sample runtimeobs.Sample - sample, err = observeProvider(ctx, p, *q.Observation) + sample, err = p.Observe(ctx, *q.Observation) out.Sample = &sample case "create": info, err = p.Create(ctx, *q.Bootstrap) @@ -324,38 +324,33 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) response command, err = p.RunCommand(ctx, q.Reference, *q.Command) out.Command = &command default: - cp, checkpointErr := sandbox.Checkpoint(p) - if checkpointErr != nil { - err = checkpointErr - break - } var state sandbox.ComputeState var compute sandbox.Compute switch q.Operation { case "initial": - compute, err = cp.Initial(ctx, q.Reference) + compute, err = p.Initial(ctx, q.Reference) out.Compute = &compute case "new_compute": - compute, err = cp.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) + compute, err = p.NewCompute(ctx, q.Reference, q.Generation, q.Snapshot) out.Compute = &compute case "compute": - state, err = cp.GetCompute(ctx, q.Reference, *q.Compute) + state, err = p.GetCompute(ctx, q.Reference, *q.Compute) out.State = &state case "suspend": - state, err = cp.Suspend(ctx, *q.Suspend) + state, err = p.Suspend(ctx, *q.Suspend) out.State = &state case "resume": - state, err = cp.Resume(ctx, *q.Resume) + state, err = p.Resume(ctx, *q.Resume) out.State = &state case "kill_compute": - err = cp.KillCompute(ctx, q.Reference, *q.Compute) + err = p.KillCompute(ctx, q.Reference, *q.Compute) case "delete_snapshot": - err = cp.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) + err = p.DeleteSnapshot(ctx, q.Reference, *q.Snapshot) case "resume_compute": - state, err = cp.ResumeCompute(ctx, q.Reference, *q.Compute) + state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state case "command_compute": - command, err = cp.RunCommandCompute(ctx, q.Reference, *q.Compute, *q.Command) + command, err = p.RunCommandCompute(ctx, q.Reference, *q.Compute, *q.Command) out.Command = &command default: err = sandbox.ErrInvalid diff --git a/services/core/internal/sandbox/operations.go b/services/core/internal/sandbox/operations.go deleted file mode 100644 index 3faf4454b..000000000 --- a/services/core/internal/sandbox/operations.go +++ /dev/null @@ -1,88 +0,0 @@ -package sandbox - -import ( - "errors" - "fmt" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" - "reflect" -) - -// These existing interfaces are the canonical operation inventory. Declarations -// must cover every method, including explicit unsupported implementations. -var providerInterfaces = []reflect.Type{ - reflect.TypeFor[SandboxProvider](), reflect.TypeFor[CheckpointProvider](), - reflect.TypeFor[SelectionDiscoverer](), reflect.TypeFor[CredentialVerifier](), - reflect.TypeFor[runtimeobs.SourceResolver](), reflect.TypeFor[runtimeobs.Source](), reflect.TypeFor[runtimeobs.BatchSource](), -} - -func ValidateProvider(p SandboxProvider) error { - if err := providercontract.Validate(p, providerInterfaces...); err != nil { - return err - } - if err := runtimeobs.ValidateSource(p.(runtimeobs.Source)); err != nil { - return err - } - return ValidateOperations(p.ProviderOperations()) -} - -// ValidateOperations rejects omitted, unknown and contradictory declarations. -func ValidateOperations(operations providercontract.Operations) error { - known := map[string]bool{} - for _, contract := range providerInterfaces { - for i := 0; i < contract.NumMethod(); i++ { - name := contract.Method(i).Name - if name != "ProviderOperations" { - known[name] = true - if err := operations[name].Check(name); err != nil && !errors.Is(err, providercontract.ErrUnsupported) { - return err - } - } - } - } - for name := range operations { - if !known[name] { - return fmt.Errorf("%w: unknown operation %s", providercontract.ErrContract, name) - } - } - required := reflect.TypeFor[SandboxProvider]() - for i := 0; i < required.NumMethod(); i++ { - name := required.Method(i).Name - if name != "ProviderOperations" && operations[name].State != providercontract.Supported { - return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) - } - } - for _, name := range []string{"ObservationProviderType", "ResolveObservationSource"} { - if operations[name].State != providercontract.Supported { - return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) - } - } - // The checkpoint lifecycle is indivisible: partial cleanup or restore support - // cannot safely own a compute incarnation. - checkpoint := reflect.TypeFor[CheckpointProvider]() - for i := 0; i < checkpoint.NumMethod(); i++ { - name := checkpoint.Method(i).Name - if _, required := reflect.TypeFor[SandboxProvider]().MethodByName(name); !required && operations[name].State != operations["Initial"].State { - return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) - } - } - if operations["ObserveBatch"].State == providercontract.Supported && operations["Observe"].State != providercontract.Supported { - return fmt.Errorf("%w: batch observation requires observation", providercontract.ErrContract) - } - return nil -} - -func SupportsCheckpoint(p SandboxProvider) bool { - return providercontract.Require(p, "Initial") == nil -} - -func Checkpoint(p SandboxProvider) (CheckpointProvider, error) { - if err := providercontract.Require(p, "Initial"); err != nil { - return nil, err - } - cp, ok := p.(CheckpointProvider) - if !ok { - return nil, providercontract.ErrContract - } - return cp, nil -} diff --git a/services/core/internal/sandbox/operations_test.go b/services/core/internal/sandbox/operations_test.go index f1cc091d1..62d2c5b3f 100644 --- a/services/core/internal/sandbox/operations_test.go +++ b/services/core/internal/sandbox/operations_test.go @@ -19,23 +19,13 @@ type changedDeclaration struct { func (p *changedDeclaration) ProviderOperations() providercontract.Operations { return p.operations } -type onlyRequired struct{ sandbox.SandboxProvider } - -func (*onlyRequired) ProviderOperations() providercontract.Operations { return docker.Operations() } - func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T) { for _, mutate := range []struct { name string change func(providercontract.Operations) }{ - {"identity unsupported", func(o providercontract.Operations) { - o["ObservationProviderType"] = providercontract.Support{State: providercontract.Unsupported, Reason: "no_identity"} - }}, - {"resolver unsupported", func(o providercontract.Operations) { - o["ResolveObservationSource"] = providercontract.Support{State: providercontract.Unsupported, Reason: "no_resolver"} - }}, {"omitted", func(o providercontract.Operations) { delete(o, "DeleteSnapshot") }}, - {"zero", func(o providercontract.Operations) { o["ObserveBatch"] = providercontract.Support{} }}, + {"zero", func(o providercontract.Operations) { o["Observe"] = providercontract.Support{} }}, {"unknown", func(o providercontract.Operations) { o["FutureOperation"] = providercontract.Support{State: providercontract.Supported} }}, @@ -46,10 +36,10 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T o["Initial"] = providercontract.Support{State: providercontract.Supported} }}, {"unsafe reason", func(o providercontract.Operations) { - o["ObserveBatch"] = providercontract.Support{State: providercontract.Unsupported, Reason: "https://private:key@host"} + o["Observe"] = providercontract.Support{State: providercontract.Unsupported, Reason: "https://private:key@host"} }}, {"unknown state", func(o providercontract.Operations) { - o["ObserveBatch"] = providercontract.Support{State: "unavailable"} + o["Observe"] = providercontract.Support{State: "unavailable"} }}, } { t.Run(mutate.name, func(t *testing.T) { @@ -60,9 +50,6 @@ func TestDeclarationsRejectMissingUnknownAndContradictoryOperations(t *testing.T } }) } - if err := sandbox.ValidateProvider(&onlyRequired{}); !errors.Is(err, providercontract.ErrContract) { - t.Fatal("missing extension implementations accepted", err) - } var nilProvider *docker.Provider if err := sandbox.ValidateProvider(nilProvider); !errors.Is(err, providercontract.ErrContract) { t.Fatal("typed nil accepted", err) @@ -100,28 +87,3 @@ func TestEveryUnsupportedNativeOperationRejectsWithoutSideEffects(t *testing.T) } } } - -// An extended interface cannot inherit success through the existing declaration. -type nextContract interface { - sandbox.SandboxProvider - NextOperation(context.Context) error -} -type futureProvider struct{ *docker.Provider } - -func (*futureProvider) NextOperation(context.Context) error { return nil } -func TestNewContractRequiresAnAuthoredDecision(t *testing.T) { - for _, p := range []providercontract.Declared{&docker.Provider{}, &futureProvider{&docker.Provider{}}} { - if err := providercontract.Validate(p, reflect.TypeFor[nextContract]()); !errors.Is(err, providercontract.ErrContract) { - t.Fatal("new operation inherited a default", err) - } - } -} - -type invalidObservationIdentity struct{ *docker.Provider } - -func (*invalidObservationIdentity) ObservationProviderType() string { return "" } -func TestProviderRegistrationRequiresObservationIdentity(t *testing.T) { - if err := sandbox.ValidateProvider(&invalidObservationIdentity{&docker.Provider{}}); !errors.Is(err, providercontract.ErrContract) { - t.Fatal("provider registration accepted empty observation identity", err) - } -} diff --git a/services/core/internal/sandbox/providers/config.go b/services/core/internal/sandbox/providers/config.go index 22e789f49..32a3b1f33 100644 --- a/services/core/internal/sandbox/providers/config.go +++ b/services/core/internal/sandbox/providers/config.go @@ -64,6 +64,8 @@ type Built struct { Provider sandbox.SandboxProvider InstallationID, BackendFingerprint string Probe func(context.Context) error + // Quiescent is nil when no helper can outlive its caller. + Quiescent func() bool } // LocalOptions supplies process-local context without changing persisted configuration. diff --git a/services/core/internal/sandbox/providers/configuration.go b/services/core/internal/sandbox/providers/configuration.go index 910a0cb4c..7658c0668 100644 --- a/services/core/internal/sandbox/providers/configuration.go +++ b/services/core/internal/sandbox/providers/configuration.go @@ -3,6 +3,8 @@ package providers import ( "context" "encoding/json" + "errors" + "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -106,11 +108,48 @@ func (r *Registry) DiscoverConfiguration(ctx context.Context, kind string, input if err := a.Configuration.Requirements().Discovery.Check("DiscoverConfiguration"); err != nil { return nil, err } - discovery, ok := a.Configuration.(sandbox.ConfigurationDiscoverer) - if !ok { - return nil, providercontract.ErrContract + result, err := a.Configuration.DiscoverConfiguration(ctx, input, paths) + if err != nil { + return nil, declaredResult("DiscoverConfiguration", err) + } + return result, nil +} + +// DiscoverSelection resolves a direct candidate's omitted native values. +func (r *Registry) DiscoverSelection(ctx context.Context, c sandbox.DirectConfig) (sandbox.Selection, error) { + a, err := r.Lookup(c.Selection.Provider) + if err != nil { + return sandbox.Selection{}, err + } + if err := a.Configuration.Requirements().SelectionDiscovery.Check("DiscoverSelection"); err != nil { + return sandbox.Selection{}, err + } + s, err := a.Configuration.DiscoverSelection(ctx, c) + if err != nil { + return sandbox.Selection{}, declaredResult("DiscoverSelection", err) + } + return s, nil +} + +// VerifyCredential checks a candidate credential against owned resources. +func (r *Registry) VerifyCredential(ctx context.Context, c sandbox.DirectConfig, refs []sandbox.Reference) error { + a, err := r.Lookup(c.Selection.Provider) + if err != nil { + return err } - return discovery.DiscoverConfiguration(ctx, input, paths) + if err := a.Configuration.Requirements().CredentialVerification.Check("VerifyCredential"); err != nil { + return err + } + return declaredResult("VerifyCredential", a.Configuration.VerifyCredential(ctx, c, refs)) +} + +// declaredResult keeps a supported declaration binding: an operation declared +// Supported that reports Unsupported breaks the contract. +func declaredResult(operation string, err error) error { + if errors.Is(err, providercontract.ErrUnsupported) { + return fmt.Errorf("%w: %s is declared supported", providercontract.ErrContract, operation) + } + return err } type nodeConfiguration struct{} @@ -123,7 +162,10 @@ type nodeConfigurationAdapter struct { } func (nodeConfigurationAdapter) Requirements() sandbox.ConfigurationRequirements { - return sandbox.ConfigurationRequirements{Credential: sandbox.NotRequired, PublicOrigin: sandbox.NotRequired, Discovery: providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_has_no_catalog"}} + return sandbox.ConfigurationRequirements{Credential: sandbox.NotRequired, PublicOrigin: sandbox.NotRequired, + Discovery: providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_has_no_catalog"}, + SelectionDiscovery: providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_has_no_catalog"}, + CredentialVerification: providercontract.Support{State: providercontract.Unsupported, Reason: "credentials_not_required"}} } func (nodeConfigurationAdapter) DecodeInput(public, secret json.RawMessage) (sandbox.Configuration, error) { if len(secret) > 0 || sandbox.DecodeConfigurationObject(public, &struct{}{}) != nil { @@ -170,3 +212,9 @@ func (a nodeConfigurationAdapter) Equal(x, y sandbox.Configuration) (bool, error func (nodeConfigurationAdapter) DiscoverConfiguration(context.Context, sandbox.ConfigurationDiscoveryInput, sandbox.ProcessPaths) (json.RawMessage, error) { return nil, &providercontract.UnsupportedError{Operation: "DiscoverConfiguration", Reason: "node_configuration_has_no_catalog"} } +func (nodeConfigurationAdapter) DiscoverSelection(context.Context, sandbox.DirectConfig) (sandbox.Selection, error) { + return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "node_configuration_has_no_catalog"} +} +func (nodeConfigurationAdapter) VerifyCredential(context.Context, sandbox.DirectConfig, []sandbox.Reference) error { + return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "credentials_not_required"} +} diff --git a/services/core/internal/sandbox/providers/configuration_flow_test.go b/services/core/internal/sandbox/providers/configuration_flow_test.go index b61b73c5a..12b338067 100644 --- a/services/core/internal/sandbox/providers/configuration_flow_test.go +++ b/services/core/internal/sandbox/providers/configuration_flow_test.go @@ -31,7 +31,8 @@ func (regionalConfiguration) ReplacesCredential() bool { return false } type regionalCodec struct{} func (regionalCodec) Requirements() sandbox.ConfigurationRequirements { - return sandbox.ConfigurationRequirements{Credential: sandbox.NotRequired, PublicOrigin: sandbox.NotRequired, Discovery: providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_has_no_catalog"}} + unsupported := providercontract.Support{State: providercontract.Unsupported, Reason: "node_configuration_has_no_catalog"} + return sandbox.ConfigurationRequirements{Credential: sandbox.NotRequired, PublicOrigin: sandbox.NotRequired, Discovery: unsupported, SelectionDiscovery: unsupported, CredentialVerification: unsupported} } func (regionalCodec) WithCredential(sandbox.Configuration, sandbox.Configuration) (sandbox.Configuration, error) { return nil, &providercontract.UnsupportedError{Operation: "WithCredential", Reason: "credentials_not_required"} @@ -75,6 +76,12 @@ func (a regionalCodec) Equal(x, y sandbox.Configuration) (bool, error) { func (regionalCodec) DiscoverConfiguration(context.Context, sandbox.ConfigurationDiscoveryInput, sandbox.ProcessPaths) (json.RawMessage, error) { return nil, &providercontract.UnsupportedError{Operation: "DiscoverConfiguration", Reason: "node_configuration_has_no_catalog"} } +func (regionalCodec) DiscoverSelection(context.Context, sandbox.DirectConfig) (sandbox.Selection, error) { + return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "node_configuration_has_no_catalog"} +} +func (regionalCodec) VerifyCredential(context.Context, sandbox.DirectConfig, []sandbox.Reference) error { + return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "node_configuration_has_no_catalog"} +} // A registered native configuration reaches the ordinary API and Store without // adding its fields or kind to either Core package. diff --git a/services/core/internal/sandbox/providers/configuration_test.go b/services/core/internal/sandbox/providers/configuration_test.go index 43588e61a..cfabbadeb 100644 --- a/services/core/internal/sandbox/providers/configuration_test.go +++ b/services/core/internal/sandbox/providers/configuration_test.go @@ -25,11 +25,7 @@ func TestNodeConfigurationExplicitUnsupportedAndStrictEmptyInput(t *testing.T) { t.Fatal(kind, "accepted credential", err) } } - discover, ok := a.Configuration.(sandbox.ConfigurationDiscoverer) - if !ok { - t.Fatal("missing explicit discovery implementation") - } - if _, err := discover.DiscoverConfiguration(t.Context(), sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}); !errors.Is(err, providercontract.ErrUnsupported) { + if _, err := a.Configuration.DiscoverConfiguration(t.Context(), sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}); !errors.Is(err, providercontract.ErrUnsupported) { t.Fatal("discovery did not reject", err) } if _, err := a.Configuration.WithCredential(nil, nil); !errors.Is(err, providercontract.ErrUnsupported) { diff --git a/services/core/internal/sandbox/providers/e2b.go b/services/core/internal/sandbox/providers/e2b.go deleted file mode 100644 index bb3861f7f..000000000 --- a/services/core/internal/sandbox/providers/e2b.go +++ /dev/null @@ -1,56 +0,0 @@ -package providers - -import ( - "errors" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" -) - -type DirectConfig struct { - ProcessPaths sandbox.ProcessPaths - InstallationID string - Selection sandbox.Selection - Fence *sandbox.CallFence -} - -func (r *Registry) BuildDirect(c DirectConfig) (sandbox.SandboxProvider, error) { - a, e := r.Lookup(c.Selection.Provider) - if e != nil { - return nil, e - } - if a.BuildDirect == nil { - return nil, sandbox.ErrInvalid - } - p, err := a.BuildDirect(c) - if err != nil { - return nil, err - } - if err := ValidateBinding(a, p); err != nil { - return nil, err - } - return p, nil -} -func buildE2B(c DirectConfig) (sandbox.SandboxProvider, error) { - configuration, ok := c.Selection.Configuration.(*e2b.DeploymentConfiguration) - if !ok || configuration == nil { - return nil, errors.New("E2B deployment configuration is unavailable") - } - binary, state, err := e2b.InstalledPaths(c.ProcessPaths) - if err != nil { - return nil, err - } - // Only a candidate that omitted its resources has none; its validation - // reads them from the template build before the candidate is rebuilt. - var resources *sandbox.Resources - if c.Selection.DeploymentSpec.Resources != (sandbox.Resources{}) { - resources = &c.Selection.DeploymentSpec.Resources - } - provider, err := e2b.NewWithCaller(e2b.Config{Binary: binary, StateDir: state, - Resources: resources, InstallationID: c.InstallationID, APIKey: configuration.APIKey, Template: configuration.Template, - APIURL: configuration.APIURL, Domain: configuration.Domain, TimeoutSeconds: 3600}, &e2b.ProcessCaller{Fence: c.Fence}) - if err != nil { - return nil, errors.New("E2B provider cannot load; check the installed helper and private state directory") - } - return provider, nil -} diff --git a/services/core/internal/sandbox/providers/generation_test.go b/services/core/internal/sandbox/providers/generation_test.go index 58f961693..e9b1c957c 100644 --- a/services/core/internal/sandbox/providers/generation_test.go +++ b/services/core/internal/sandbox/providers/generation_test.go @@ -81,6 +81,9 @@ func TestMicrosandboxGenerationBindsLeaseIdentity(t *testing.T) { t.Fatal(err) } defer closeProvider() + if built.Quiescent == nil || !built.Quiescent() { + t.Fatal("generation does not report helper quiescence") + } response, err := json.Marshal(sandboxmicro.Response{Version: sandboxmicro.ProtocolVersion}) if err != nil { t.Fatal(err) diff --git a/services/core/internal/sandbox/providers/microsandbox.go b/services/core/internal/sandbox/providers/microsandbox.go index 06dc7c3a4..58fbcb8cf 100644 --- a/services/core/internal/sandbox/providers/microsandbox.go +++ b/services/core/internal/sandbox/providers/microsandbox.go @@ -59,6 +59,7 @@ func configureMicrosandbox(entry Microsandbox, resources sandbox.Resources, call } result.Provider = provider result.Probe = microsandboxProbe(entry, resources) + result.Quiescent = caller.Quiescent result.BackendFingerprint = BackendFingerprint("microsandbox", entry.RuntimeHome) return nil } diff --git a/services/core/internal/sandbox/providers/registration_configuration.go b/services/core/internal/sandbox/providers/registration_configuration.go index 3d8a5dd54..07ba5d285 100644 --- a/services/core/internal/sandbox/providers/registration_configuration.go +++ b/services/core/internal/sandbox/providers/registration_configuration.go @@ -24,32 +24,13 @@ func validateConfigurationAdapter(configuration sandbox.ConfigurationAdapter) er return configurationRegistrationError() } } - discovery := reflect.TypeFor[sandbox.ConfigurationDiscoverer]() - if !value.Type().Implements(discovery) { - return configurationRegistrationError() - } - if err := validateConfigurationDiscoveryInterface(discovery); err != nil { - return err - } return validateConfigurationRequirements(reflect.ValueOf(configuration.Requirements())) } -// Every discovery method needs its own authored requirement. A future method -// cannot inherit the existing Discovery decision merely by being implemented. -func validateConfigurationDiscoveryInterface(discovery reflect.Type) error { - if discovery.NumMethod() != 1 { - return configurationRegistrationError() - } - if _, exists := discovery.MethodByName("DiscoverConfiguration"); !exists { - return configurationRegistrationError() - } - return nil -} - // Check field names as well as values so new requirements cannot bypass the // gate. This owns only configuration requirements, not resource operations. func validateConfigurationRequirements(value reflect.Value) error { - if value.Kind() != reflect.Struct || value.NumField() != 3 { + if value.Kind() != reflect.Struct || value.NumField() != 5 { return configurationRegistrationError() } for i := 0; i < value.NumField(); i++ { @@ -59,12 +40,12 @@ func validateConfigurationRequirements(value reflect.Value) error { if !ok || (requirement != sandbox.Required && requirement != sandbox.NotRequired) { return configurationRegistrationError() } - case "Discovery": + case "Discovery", "SelectionDiscovery", "CredentialVerification": support, ok := value.Field(i).Interface().(providercontract.Support) if !ok { return configurationRegistrationError() } - if err := support.Check("DiscoverConfiguration"); err != nil && !errors.Is(err, providercontract.ErrUnsupported) { + if err := support.Check(value.Type().Field(i).Name); err != nil && !errors.Is(err, providercontract.ErrUnsupported) { return configurationRegistrationError() } default: diff --git a/services/core/internal/sandbox/providers/registration_configuration_test.go b/services/core/internal/sandbox/providers/registration_configuration_test.go index 7fdcda590..4a456d31e 100644 --- a/services/core/internal/sandbox/providers/registration_configuration_test.go +++ b/services/core/internal/sandbox/providers/registration_configuration_test.go @@ -2,6 +2,7 @@ package providers import ( "context" + "encoding/json" "errors" "reflect" "testing" @@ -14,7 +15,6 @@ import ( // call through the nil embedded interfaces fails the test immediately. type registrationConfiguration struct { sandbox.ConfigurationAdapter - sandbox.ConfigurationDiscoverer requirements sandbox.ConfigurationRequirements } @@ -22,17 +22,11 @@ func (a registrationConfiguration) Requirements() sandbox.ConfigurationRequireme return a.requirements } -// A declaration of Unsupported still requires an explicit rejection method. -type missingConfigurationDiscovery struct{ sandbox.ConfigurationAdapter } - -func TestConfigurationRegistrationRejectsNilAndMissingDiscovery(t *testing.T) { +func TestConfigurationRegistrationRejectsNil(t *testing.T) { registry := Builtin() a := registry.adapters["docker"] var typedNil *registrationConfiguration - for _, configuration := range []sandbox.ConfigurationAdapter{ - nil, typedNil, missingConfigurationDiscovery{a.Configuration}, - missingConfigurationDiscovery{registry.adapters["e2b"].Configuration}, - } { + for _, configuration := range []sandbox.ConfigurationAdapter{nil, typedNil} { a.Configuration = configuration if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) { t.Fatalf("%T: %v", configuration, err) @@ -104,12 +98,7 @@ func TestConfigurationRequirementsDoNotInventDependencies(t *testing.T) { } } -type futureConfigurationDiscovery interface { - sandbox.ConfigurationDiscoverer - NextDiscovery(context.Context) error -} - -func TestFutureConfigurationRequirementAndMethodNeedExplicitHandling(t *testing.T) { +func TestFutureConfigurationRequirementNeedsExplicitHandling(t *testing.T) { registry := Builtin() original := registry.adapters["docker"].Configuration.Requirements() fields := make([]reflect.StructField, 0, 4) @@ -126,32 +115,85 @@ func TestFutureConfigurationRequirementAndMethodNeedExplicitHandling(t *testing. if err := validateConfigurationRequirements(value); !errors.Is(err, providercontract.ErrContract) { t.Fatal("new requirement silently inherited policy", err) } - if err := validateConfigurationDiscoveryInterface(reflect.TypeFor[futureConfigurationDiscovery]()); !errors.Is(err, providercontract.ErrContract) { - t.Fatal("new discovery method inherited support", err) - } } -func TestUnsupportedConfigurationDiscoveryMatchesAuthoredReason(t *testing.T) { +func TestUnsupportedSetupOperationsMatchAuthoredReasons(t *testing.T) { registry := Builtin() for kind, a := range registry.adapters { - support := a.Configuration.Requirements().Discovery - if support.State != providercontract.Unsupported { - continue + requirements := a.Configuration.Requirements() + direct := sandbox.DirectConfig{Selection: sandbox.Selection{Provider: kind}} + discover := func(read func() (json.RawMessage, error)) func() error { + return func() error { + if result, err := read(); result != nil { + return errors.New("fabricated catalog") + } else { + return err + } + } } - native := a.Configuration.(sandbox.ConfigurationDiscoverer) - for _, read := range []func() ([]byte, error){ - func() ([]byte, error) { - return native.DiscoverConfiguration(t.Context(), sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) - }, - func() ([]byte, error) { - return registry.DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) - }, + for _, operation := range []struct { + name string + support providercontract.Support + calls []func() error + }{ + {"DiscoverConfiguration", requirements.Discovery, []func() error{ + discover(func() (json.RawMessage, error) { + return a.Configuration.DiscoverConfiguration(t.Context(), sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) + }), + discover(func() (json.RawMessage, error) { + return registry.DiscoverConfiguration(t.Context(), kind, sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) + }), + }}, + {"DiscoverSelection", requirements.SelectionDiscovery, []func() error{ + func() error { _, err := a.Configuration.DiscoverSelection(t.Context(), direct); return err }, + func() error { _, err := registry.DiscoverSelection(t.Context(), direct); return err }, + }}, + {"VerifyCredential", requirements.CredentialVerification, []func() error{ + func() error { return a.Configuration.VerifyCredential(t.Context(), direct, nil) }, + func() error { return registry.VerifyCredential(t.Context(), direct, nil) }, + }}, } { - result, err := read() - reason, valid := providercontract.UnsupportedReason(err, "DiscoverConfiguration") - if result != nil || !valid || reason != support.Reason { - t.Fatalf("%s: result=%v reason=%s err=%v", kind, result, reason, err) + if operation.support.State != providercontract.Unsupported { + continue + } + for _, call := range operation.calls { + err := call() + if reason, valid := providercontract.UnsupportedReason(err, operation.name); !valid || reason != operation.support.Reason { + t.Fatalf("%s %s: reason=%s err=%v", kind, operation.name, reason, err) + } } } } } + +// contradictingConfiguration declares every setup operation Supported and then +// reports each as Unsupported. +type contradictingConfiguration struct{ registrationConfiguration } + +func (contradictingConfiguration) DiscoverConfiguration(context.Context, sandbox.ConfigurationDiscoveryInput, sandbox.ProcessPaths) (json.RawMessage, error) { + return nil, &providercontract.UnsupportedError{Operation: "DiscoverConfiguration", Reason: "not_ready"} +} +func (contradictingConfiguration) DiscoverSelection(context.Context, sandbox.DirectConfig) (sandbox.Selection, error) { + return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "not_ready"} +} +func (contradictingConfiguration) VerifyCredential(context.Context, sandbox.DirectConfig, []sandbox.Reference) error { + return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "not_ready"} +} + +func TestSupportedSetupOperationsCannotReportUnsupported(t *testing.T) { + registry := Builtin() + a := registry.adapters["docker"] + requirements := a.Configuration.Requirements() + supported := providercontract.Support{State: providercontract.Supported} + requirements.Discovery, requirements.SelectionDiscovery, requirements.CredentialVerification = supported, supported, supported + a.Configuration = contradictingConfiguration{registrationConfiguration{requirements: requirements}} + registry.adapters["docker"] = a + direct := sandbox.DirectConfig{Selection: sandbox.Selection{Provider: "docker"}} + _, discoverErr := registry.DiscoverConfiguration(t.Context(), "docker", sandbox.ConfigurationDiscoveryInput{}, sandbox.ProcessPaths{}) + _, selectionErr := registry.DiscoverSelection(t.Context(), direct) + for _, err := range []error{discoverErr, selectionErr, registry.VerifyCredential(t.Context(), direct, nil)} { + if !errors.Is(err, providercontract.ErrContract) || errors.Is(err, providercontract.ErrUnsupported) { + t.Fatal(err) + } + } +} diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 93eb5304b..f30c39072 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -38,7 +38,7 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { {"wrong local constructor", func(a *Adapter) { a.Mode = "direct" }}, {"missing direct constructor", func(a *Adapter) { a.Mode = "direct"; a.BuildLocal = nil }}, {"both constructors", func(a *Adapter) { - a.BuildDirect = func(DirectConfig) (sandbox.SandboxProvider, error) { + a.BuildDirect = func(sandbox.DirectConfig) (sandbox.SandboxProvider, error) { t.Fatal("called direct constructor") return nil, nil } @@ -47,7 +47,6 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { {"missing resource validator", func(a *Adapter) { a.ValidateResources = nil }}, {"missing configuration", func(a *Adapter) { a.Configuration = nil }}, {"typed nil configuration", func(a *Adapter) { var c *registrationConfiguration; a.Configuration = c }}, - {"missing discovery implementation", func(a *Adapter) { a.Configuration = missingConfigurationDiscovery{a.Configuration} }}, {"missing configuration requirement", func(a *Adapter) { a.Configuration = registrationConfiguration{} }}, {"invalid credential requirement", func(a *Adapter) { r := a.Configuration.Requirements() @@ -100,7 +99,7 @@ func TestRegistrationRejectsBeforeCallbacksOrConstruction(t *testing.T) { _, _, err := registry.Build(Config{Provider: kind, Generation: 1, InstallationID: uuid.NewString(), Specification: selection.DeploymentSpec}, LocalOptions{Standalone: true}) return err }}, - {"direct build", func() error { _, err := registry.BuildDirect(DirectConfig{Selection: selection}); return err }}, + {"direct build", func() error { _, err := registry.BuildDirect(sandbox.DirectConfig{Selection: selection}); return err }}, {"binding", func() error { return ValidateBinding(a, &docker.Provider{}) }}, {"projection", func() error { text, err := registry.PythonDeploymentContract() @@ -152,12 +151,12 @@ func TestCompleteRegistrationsPreserveConstruction(t *testing.T) { // Direct providers may legitimately need no remote credential or extra // selection state; registration must not require irrelevant callback stubs. a.Mode, a.BuildLocal, a.NodeArtifacts = "direct", nil, nil - a.BuildDirect = func(DirectConfig) (sandbox.SandboxProvider, error) { + a.BuildDirect = func(sandbox.DirectConfig) (sandbox.SandboxProvider, error) { calls++ return &docker.Provider{}, nil } registry.adapters[kind] = a - p, err := registry.BuildDirect(DirectConfig{Selection: sandbox.Selection{Provider: kind}}) + p, err := registry.BuildDirect(sandbox.DirectConfig{Selection: sandbox.Selection{Provider: kind}}) if err != nil || p == nil || calls != 2 { t.Fatalf("credential-free direct build: %v calls=%d", err, calls) } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index 5d60ab7fc..29bb86b4f 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -21,7 +21,7 @@ type Adapter struct { Policy sandbox.DeploymentPolicy Configuration sandbox.ConfigurationAdapter BuildLocal func(Config, LocalOptions, *Built) (func(), error) - BuildDirect func(DirectConfig) (sandbox.SandboxProvider, error) + BuildDirect func(sandbox.DirectConfig) (sandbox.SandboxProvider, error) Mode string Operations func() providercontract.Operations IdleSeconds, RetentionSeconds int64 @@ -56,7 +56,7 @@ func Builtin() *Registry { Configuration: nodeConfigurationAdapter{microsandbox.ValidateSpecification}, }, "e2b": { - Policy: e2b.Policy(), Operations: e2b.Operations, Mode: "direct", BuildDirect: buildE2B, + Policy: e2b.Policy(), Operations: e2b.Operations, Mode: "direct", BuildDirect: e2b.BuildDirect, Configuration: e2b.ConfigurationAdapter{}, ValidateSpecification: e2b.ValidateSpecification, ValidateResources: e2b.ValidateResources, }, @@ -75,13 +75,23 @@ func (r *Registry) Lookup(kind string) (Adapter, error) { return a, nil } -// IsNode reports whether the provider runs on enrolled sandbox nodes. -func (r *Registry) IsNode(kind string) (bool, error) { - a, err := r.Lookup(kind) +// BuildDirect builds a direct-mode Provider and validates its binding. +func (r *Registry) BuildDirect(c sandbox.DirectConfig) (sandbox.SandboxProvider, error) { + a, e := r.Lookup(c.Selection.Provider) + if e != nil { + return nil, e + } + if a.BuildDirect == nil { + return nil, sandbox.ErrInvalid + } + p, err := a.BuildDirect(c) if err != nil { - return false, err + return nil, err + } + if err := ValidateBinding(a, p); err != nil { + return nil, err } - return a.Mode == "nodes", nil + return p, nil } // SupportsCheckpoint reports whether the provider declares checkpoint suspension. diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 97c1539df..6b3078902 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -28,9 +28,8 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { } a, err := registry.Lookup(tc.kind) checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) - isNode, nodeErr := registry.IsNode(tc.kind) - if err != nil || checkpointErr != nil || nodeErr != nil || checkpoint != tc.checkpoint || isNode != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { - t.Fatal("inconsistent construction/capability registration", err, checkpointErr, nodeErr) + if err != nil || checkpointErr != nil || checkpoint != tc.checkpoint || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { + t.Fatal("inconsistent construction/capability registration", err, checkpointErr) } }) } @@ -71,10 +70,9 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { // Registration is test-local: production registrations are fixed, never plugins. registry.adapters[kind] = registry.adapters["docker"] s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - isNode, nodeErr := registry.IsNode(kind) checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) - if err != nil || nodeErr != nil || checkpointErr != nil || s.Provider != kind || !isNode || checkpoint { - t.Fatal("new entry did not follow shared boundary", err, nodeErr, checkpointErr) + if err != nil || checkpointErr != nil || s.Provider != kind || checkpoint { + t.Fatal("new entry did not follow shared boundary", err, checkpointErr) } d, err := registry.Describe(kind, uuid.NewString()) if err != nil || d.Mode != "nodes" || d.IdleSeconds != 0 { @@ -113,9 +111,6 @@ func TestCapabilityLookupsReportFailures(t *testing.T) { invalid.Operations = nil registry.adapters["invalid-registration"] = invalid for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { - if _, err := registry.IsNode(kind); !errors.Is(err, want) { - t.Fatal(kind, "IsNode", err) - } if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { t.Fatal(kind, "SupportsCheckpoint", err) } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 663526b8e..aac367557 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -1,27 +1,36 @@ // Package sandbox defines the Sandbox Provider contract for authorized compute. -// Start at sandbox_provider.go and docs/sandbox-provider.md when adding an adapter. +// This file is the Core–Sandbox Provider protocol; docs/sandbox-provider.md +// describes it. Start here when adding an adapter. Value types that Core also +// uses beyond this boundary, such as DeploymentSpec and CallFence, live in their +// own files of this package. // Core owns durable Environment/allocation state and lifecycle serialization; // SandboxProvider owns compute and bootstrap, Runtime owns capability preparation, // and Harness adapters own native execution. Compute running is not execution ready. // -// Required operations are on SandboxProvider. CheckpointProvider and runtimeobs -// observation remain separate small interfaces. Every registered adapter explicitly -// declares and implements each operation, including safe Unsupported rejections. -// Method-set presence never means an extension is supported. ValidateProvider and -// the common contract tests check declaration completeness and implementation. +// The protocol has two interfaces. SandboxProvider is the allocation-time half +// and ConfigurationAdapter the setup-time half. Every adapter implements every +// method of both and declares which operations it supports: SandboxProvider +// through ProviderOperations, ConfigurationAdapter through Requirements. An +// unsupported method returns a typed providercontract.UnsupportedError; method-set +// presence never means support. // -// Registration is explicit construction, not a global init-time registry. Node-local -// adapters register in sandbox/providers; Core's managed setup -// constructs direct adapters or node proxies. execution.RuntimeProvider binds the -// selected adapter to installation, backend, deployment generation and node identity. -// Keep vendor configuration at those construction boundaries; common lifecycle code -// selects behavior through these contracts, never through a vendor name. +// Registration is explicit construction, not a global init-time registry. Adapters +// register in sandbox/providers; Core's managed setup constructs direct adapters +// or node proxies. execution.RuntimeProvider binds the selected adapter to +// installation, backend, deployment generation and node identity. Keep vendor +// configuration at those construction boundaries; common lifecycle code selects +// behavior through these contracts, never through a vendor name. package sandbox import ( "context" + "encoding/json" "errors" + "fmt" + "reflect" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" ) var ( @@ -30,6 +39,9 @@ var ( ErrExists = errors.New("sandbox allocation already exists") ErrNotFound = errors.New("sandbox allocation not found") ErrCommandUnconfirmed = errors.New("initialization command outcome unconfirmed; reclaim allocation before reuse") + // ErrComputeUnconfirmed requires observation of the retained operation identity; + // it does not authorize another Create, capture, restore, or cold start. + ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") ) // Reference must be persisted by the caller before Create. AllocationID is a fresh @@ -90,12 +102,9 @@ type SandboxProvider interface { // idempotent, but nil alone cannot settle an outstanding Create. Kill(context.Context, Reference) error RunCommand(context.Context, Reference, Command) (CommandResult, error) -} -// CheckpointProvider is an explicitly declared extension. It supplies -// exact-incarnation operations; Worker and Store remain the lifecycle owner. -type CheckpointProvider interface { - SandboxProvider + // The checkpoint lifecycle supplies exact-incarnation operations; Worker and + // Store remain the lifecycle owner. Its operations share one declaration. Initial(context.Context, Reference) (Compute, error) NewCompute(context.Context, Reference, uint64, *SnapshotIdentity) (Compute, error) GetCompute(context.Context, Reference, Compute) (ComputeState, error) @@ -106,6 +115,111 @@ type CheckpointProvider interface { RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) + + // Observe reads one owned Runtime instance after verifying its ownership. It + // never renews, restarts or stops compute. + Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) +} + +// requiredOperations are supported by every Provider. +var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunCommand"} + +// checkpointOperations are all supported or all unsupported: partial cleanup or +// restore support cannot safely own a compute incarnation. +var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} + +// Compute identifies one incarnation of an allocation. Name is provider-derived. +// ID is empty only until the original create or restore result is observed. +type Compute struct { + Generation uint64 + Name string + ID string + RestoredFrom *SnapshotIdentity +} + +// SnapshotIdentity is provider evidence from a verified full snapshot. Core +// persists it unchanged and records consumption separately; it never invents +// paths, checksums, native checkpoint fields, or source identity. +type SnapshotIdentity struct { + Reference string + ID string + Digest string + CheckpointID string + CheckpointRoot string + OperationID string + SourceGeneration uint64 + SourceName string + SourceID string +} + +type ComputeState struct { + Compute Compute + Status string + BootstrapComplete bool + Snapshot *SnapshotIdentity + SourceStopped bool +} +type SuspendRequest struct { + Reference Reference + OperationID string + Source Compute + Snapshot *SnapshotIdentity + // Recovery observes the previous attempt and never starts a new capture. + ObserveOnly bool +} +type ResumeRequest struct { + Reference Reference + OperationID string + Snapshot SnapshotIdentity + Target Compute + // Recovery observes the previous target and never starts a new restore. + ObserveOnly bool +} + +// ValidateProvider checks a constructed Provider's declaration. +func ValidateProvider(p SandboxProvider) error { + if p == nil { + return providercontract.ErrContract + } + if value := reflect.ValueOf(p); value.Kind() == reflect.Pointer && value.IsNil() { + return providercontract.ErrContract + } + return ValidateOperations(p.ProviderOperations()) +} + +// ValidateOperations rejects omitted, unknown and contradictory declarations. +// SandboxProvider's method set is the operation inventory. +func ValidateOperations(operations providercontract.Operations) error { + contract := reflect.TypeFor[SandboxProvider]() + for i := range contract.NumMethod() { + name := contract.Method(i).Name + if name == "ProviderOperations" { + continue + } + if err := operations[name].Check(name); err != nil && !errors.Is(err, providercontract.ErrUnsupported) { + return err + } + } + for name := range operations { + if _, known := contract.MethodByName(name); !known || name == "ProviderOperations" { + return fmt.Errorf("%w: unknown operation %s", providercontract.ErrContract, name) + } + } + for _, name := range requiredOperations { + if operations[name].State != providercontract.Supported { + return fmt.Errorf("%w: required operation %s", providercontract.ErrContract, name) + } + } + for _, name := range checkpointOperations { + if operations[name].State != operations["Initial"].State { + return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) + } + } + return nil +} + +func SupportsCheckpoint(p SandboxProvider) bool { + return providercontract.Require(p, "Initial") == nil } // ProcessPaths locates installed adapter helpers and their private state. The @@ -114,3 +228,123 @@ type ProcessPaths struct { ArtifactRoot string StateRoot string } + +// Requirement has no implicit default: registration must choose either value. +type Requirement string + +const ( + Required Requirement = "required" + NotRequired Requirement = "not_required" +) + +// ConfigurationRequirements is the setup-time declaration. Discovery, +// SelectionDiscovery and CredentialVerification declare DiscoverConfiguration, +// DiscoverSelection and VerifyCredential; requiring a credential does not +// promise VerifyCredential. +type ConfigurationRequirements struct { + Credential Requirement + PublicOrigin Requirement + Discovery providercontract.Support + SelectionDiscovery providercontract.Support + CredentialVerification providercontract.Support +} + +// Configuration is an adapter-owned typed value, never a request or response DTO. +// Implementations must exclude secrets from JSON and safe diagnostic output. +type Configuration interface { + HasCredential() bool + ReplacesCredential() bool +} + +// ConfigurationRecord separates public selectors, read-only observations and +// secret bytes. Store encrypts Secret with the installation and generation. +// Only adapter codecs may produce Public and Metadata; neither is input passthrough. +type ConfigurationRecord struct { + Public json.RawMessage + Metadata json.RawMessage + Secret []byte `json:"-"` +} + +// Selection is the typed deployment configuration shared by preview and commit. +type Selection struct { + DeploymentSpec + ExpectedGeneration uint64 `json:"expected_generation"` + Provider string `json:"provider"` + Configuration Configuration `json:"-"` +} + +func (s Selection) HasCredential() bool { + return s.Configuration != nil && s.Configuration.HasCredential() +} + +func (s Selection) ReplacesCredential() bool { + return s.Configuration != nil && s.Configuration.ReplacesCredential() +} + +// DirectConfig constructs a direct adapter for one selection. Fence counts the +// adapter's helper processes so that a credential commit waits for them. +type DirectConfig struct { + ProcessPaths ProcessPaths + InstallationID string + Selection Selection + Fence *CallFence +} + +// ConfigurationDiscoveryInput is a transient read-only request. Query is typed +// and validated by the adapter; it cannot select a compute mutation. +type ConfigurationDiscoveryInput struct { + Configuration json.RawMessage `json:"configuration" swaggertype:"object"` + Credential json.RawMessage `json:"credential" swaggertype:"object"` + Query json.RawMessage `json:"query" swaggertype:"object"` +} + +// ConfigurationAdapter owns all interpretation of provider configuration. +// Decode loads retained ownership without remote discovery or new-build admission. +// Normalize validates a candidate; ResolveChange first applies omitted-field +// inheritance, then normalizes. Equal compares normalized identity, excluding +// discovery metadata and explicit credential-submission intent. +// +// The three setup operations are read-only native calls, separate from compute +// and candidate admission. Each builds its own native client from its input. +type ConfigurationAdapter interface { + Requirements() ConfigurationRequirements + DecodeInput(public, credential json.RawMessage) (Configuration, error) + Encode(Configuration) (ConfigurationRecord, error) + Decode(ConfigurationRecord) (Configuration, error) + Normalize(Selection) (Selection, error) + ResolveChange(next, previous Selection) (Selection, error) + WithCredential(owner, candidate Configuration) (Configuration, error) + Equal(a, b Configuration) (bool, error) + // DiscoverConfiguration lists the native catalog a credential can use, + // without a saved deployment. + DiscoverConfiguration(context.Context, ConfigurationDiscoveryInput, ProcessPaths) (json.RawMessage, error) + // DiscoverSelection resolves a candidate's omitted native values before + // commit. Loading retained ownership never calls it. + DiscoverSelection(context.Context, DirectConfig) (Selection, error) + // VerifyCredential verifies access to already owned resources without + // mutation. Replacing a credential requires it and a shared CallFence. + VerifyCredential(context.Context, DirectConfig, []Reference) error +} + +// ConfigurationError is a fixed safe diagnostic, never SDK text or submitted data. +// Class describes the request outcome; it does not authorize replay. +type ConfigurationError struct { + Class ConfigurationErrorClass + Code, Param, Message string +} +type ConfigurationErrorClass string + +const ( + ConfigurationInvalid ConfigurationErrorClass = "invalid" + ConfigurationConflict ConfigurationErrorClass = "conflict" + ConfigurationUnconfirmed ConfigurationErrorClass = "unconfirmed" +) + +func (e *ConfigurationError) Error() string { return e.Message } + +var ( + ErrCredentialRejected = &ConfigurationError{ConfigurationInvalid, "sandbox_credential_invalid", "credential", "The sandbox provider credential was rejected."} + ErrCredentialOwnership = &ConfigurationError{ConfigurationConflict, "sandbox_credential_ownership", "credential", "The credential cannot manage the retained deployment. Reset before changing accounts."} + ErrConfigurationUnconfirmed = &ConfigurationError{ConfigurationUnconfirmed, "sandbox_verification_unconfirmed", "", "Sandbox provider verification could not be confirmed."} + ErrConfigurationSelection = &ConfigurationError{ConfigurationInvalid, "sandbox_configuration_invalid", "configuration", "Select a ready immutable provider configuration with matching resources."} +) diff --git a/services/core/internal/sandbox/sandbox_provider_test.go b/services/core/internal/sandbox/sandbox_provider_test.go new file mode 100644 index 000000000..56c23b330 --- /dev/null +++ b/services/core/internal/sandbox/sandbox_provider_test.go @@ -0,0 +1,24 @@ +package sandbox + +import ( + "reflect" + "slices" + "testing" +) + +// The operation groups partition SandboxProvider's operations, so a new method +// cannot escape the required or all-or-nothing checkpoint checks. +func TestOperationGroupsPartitionTheInterface(t *testing.T) { + contract := reflect.TypeFor[SandboxProvider]() + var methods []string + for i := range contract.NumMethod() { + if name := contract.Method(i).Name; name != "ProviderOperations" { + methods = append(methods, name) + } + } + groups := slices.Concat(requiredOperations, checkpointOperations, []string{"Observe"}) + slices.Sort(groups) + if !slices.Equal(methods, groups) { + t.Fatalf("SandboxProvider operations %v, groups %v", methods, groups) + } +} diff --git a/services/core/internal/sandbox/selection.go b/services/core/internal/sandbox/selection.go deleted file mode 100644 index 65fede7d2..000000000 --- a/services/core/internal/sandbox/selection.go +++ /dev/null @@ -1,31 +0,0 @@ -package sandbox - -import "context" - -// Selection is the typed deployment configuration shared by preview and commit. -type Selection struct { - DeploymentSpec - ExpectedGeneration uint64 `json:"expected_generation"` - Provider string `json:"provider"` - Configuration Configuration `json:"-"` -} - -func (s Selection) HasCredential() bool { - return s.Configuration != nil && s.Configuration.HasCredential() -} - -func (s Selection) ReplacesCredential() bool { - return s.Configuration != nil && s.Configuration.ReplacesCredential() -} - -// SelectionDiscoverer is an optional read-only native configuration capability. -// It resolves candidate configuration; loading retained ownership never calls it. -type SelectionDiscoverer interface { - DiscoverSelection(context.Context, Selection) (Selection, error) -} - -// CredentialVerifier verifies access to already owned resources without mutation. -// Replacing credentials requires this capability and a shared CallFence. -type CredentialVerifier interface { - VerifyCredential(context.Context, []Reference) error -} diff --git a/services/core/internal/sandbox/suspension.go b/services/core/internal/sandbox/suspension.go deleted file mode 100644 index 5a600a877..000000000 --- a/services/core/internal/sandbox/suspension.go +++ /dev/null @@ -1,57 +0,0 @@ -package sandbox - -import ( - "errors" -) - -// ErrComputeUnconfirmed requires observation of the retained operation identity; -// it does not authorize another Create, capture, restore, or cold start. -var ErrComputeUnconfirmed = errors.New("sandbox lifecycle outcome unconfirmed") - -// Compute identifies one incarnation of an allocation. Name is provider-derived. -// ID is empty only until the original create or restore result is observed. -type Compute struct { - Generation uint64 - Name string - ID string - RestoredFrom *SnapshotIdentity -} - -// SnapshotIdentity is provider evidence from a verified full snapshot. Core -// persists it unchanged and records consumption separately; it never invents -// paths, checksums, native checkpoint fields, or source identity. -type SnapshotIdentity struct { - Reference string - ID string - Digest string - CheckpointID string - CheckpointRoot string - OperationID string - SourceGeneration uint64 - SourceName string - SourceID string -} - -type ComputeState struct { - Compute Compute - Status string - BootstrapComplete bool - Snapshot *SnapshotIdentity - SourceStopped bool -} -type SuspendRequest struct { - Reference Reference - OperationID string - Source Compute - Snapshot *SnapshotIdentity - // Recovery observes the previous attempt and never starts a new capture. - ObserveOnly bool -} -type ResumeRequest struct { - Reference Reference - OperationID string - Snapshot SnapshotIdentity - Target Compute - // Recovery observes the previous target and never starts a new restore. - ObserveOnly bool -} diff --git a/services/core/internal/sessions/creation_test.go b/services/core/internal/sessions/creation_test.go index 865fee9e1..7fe34ff68 100644 --- a/services/core/internal/sessions/creation_test.go +++ b/services/core/internal/sessions/creation_test.go @@ -449,7 +449,7 @@ func TestCreateSession(t *testing.T) { tx := newCreationTx(t, hostedSession, true) tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Specification: json.RawMessage(`{}`)}) tx.createEnvironment = returns("environment") - tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1}}) + tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) tx.reservePlacement = done _, err, calls := runCreation(t, rules, tx, creationInput("openai_hosted")) want := []string{"UpsertSession create", "LockDeployment", "CreateEnvironment", "LoadNodes", "ReservePlacement node 5", "AuditCreation session:session environment:environment:session", "LoadSession"} diff --git a/services/core/migrations/000093_web_deployment_only.sql b/services/core/migrations/000093_web_deployment_only.sql new file mode 100644 index 000000000..54886e099 --- /dev/null +++ b/services/core/migrations/000093_web_deployment_only.sql @@ -0,0 +1,78 @@ +-- +goose Up +-- Web setup is the only writer of the installation, so a recorded installation +-- is Web-managed, no deployment names a process-configured local node, every +-- node has the Core address it enrolled with, and only a sandbox reset pauses +-- admission. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE local_node_id IS NOT NULL OR (installation_id IS NOT NULL AND NOT web_managed) + OR admission_paused <> (reset_clear IS NOT NULL)) + OR EXISTS (SELECT 1 FROM runtime_nodes WHERE removed_at IS NULL AND core_url = '') THEN + RAISE EXCEPTION 'Cannot upgrade: the sandbox deployment was configured outside Web setup, or a live node has no Core address. Reset the sandbox in Web, then upgrade'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE runtime_deployment + DROP CONSTRAINT runtime_deployment_identity_check, + DROP CONSTRAINT runtime_deployment_setup_check, + DROP CONSTRAINT runtime_deployment_reset_check, + DROP CONSTRAINT runtime_deployment_reset_admission_check, + DROP COLUMN web_managed, + DROP COLUMN local_node_id, + DROP COLUMN admission_paused, + ADD CONSTRAINT runtime_deployment_identity_check CHECK ( + (installation_id IS NULL AND backend_fingerprint = '') OR + (installation_id IS NOT NULL AND backend_fingerprint ~ '^[0-9a-f]{64}$') OR + (installation_id IS NOT NULL AND provider_kind = '' AND backend_fingerprint = '') + ), + ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + installation_id IS NULL OR + (provider_kind = '' AND mode = '' AND generation >= 0 AND idle_seconds = 0 AND retention_seconds = 0) OR + (provider_kind <> '' AND mode IN ('nodes','direct') AND generation > 0 AND + ((idle_seconds = 0 AND retention_seconds = 0) OR (idle_seconds > 0 AND retention_seconds > 0))) + ), + ADD CONSTRAINT runtime_deployment_reset_check CHECK ( + (reset_clear IS NULL AND reset_requested_at IS NULL AND reset_deadline_at IS NULL + AND reset_forced_at IS NULL AND reset_audit IS NULL) + OR (reset_clear IS NOT NULL AND installation_id IS NOT NULL AND provider_kind <> '' + AND reset_requested_at IS NOT NULL AND reset_audit IS NOT NULL + AND jsonb_typeof(reset_audit) = 'object' + AND ((reset_clear = 'auto' AND reset_deadline_at IS NOT NULL AND reset_forced_at IS NULL) + OR (reset_clear = 'force' AND reset_forced_at IS NOT NULL))) + ); + +-- +goose Down +ALTER TABLE runtime_deployment + DROP CONSTRAINT runtime_deployment_identity_check, + DROP CONSTRAINT runtime_deployment_setup_check, + DROP CONSTRAINT runtime_deployment_reset_check, + ADD COLUMN web_managed boolean NOT NULL DEFAULT false, + ADD COLUMN local_node_id uuid, + ADD COLUMN admission_paused boolean NOT NULL DEFAULT false; +UPDATE runtime_deployment SET web_managed = installation_id IS NOT NULL, admission_paused = reset_clear IS NOT NULL; +ALTER TABLE runtime_deployment + ADD CONSTRAINT runtime_deployment_identity_check CHECK ( + (installation_id IS NULL AND backend_fingerprint = '') OR + (installation_id IS NOT NULL AND backend_fingerprint ~ '^[0-9a-f]{64}$') OR + (web_managed AND installation_id IS NOT NULL AND provider_kind = '' AND backend_fingerprint = '') + ), + ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind = '' AND mode = '' AND generation >= 0 AND idle_seconds = 0 AND retention_seconds = 0) OR + (provider_kind <> '' AND mode IN ('nodes','direct') AND generation > 0 AND + ((idle_seconds = 0 AND retention_seconds = 0) OR (idle_seconds > 0 AND retention_seconds > 0))) + )) + ), + ADD CONSTRAINT runtime_deployment_reset_check CHECK ( + (reset_clear IS NULL AND reset_requested_at IS NULL AND reset_deadline_at IS NULL + AND reset_forced_at IS NULL AND reset_audit IS NULL) + OR (reset_clear IS NOT NULL AND web_managed AND provider_kind <> '' + AND reset_requested_at IS NOT NULL AND reset_audit IS NOT NULL + AND jsonb_typeof(reset_audit) = 'object' + AND ((reset_clear = 'auto' AND reset_deadline_at IS NOT NULL AND reset_forced_at IS NULL) + OR (reset_clear = 'force' AND reset_forced_at IS NOT NULL))) + ), + ADD CONSTRAINT runtime_deployment_reset_admission_check CHECK ( + NOT web_managed OR admission_paused = (reset_clear IS NOT NULL) + ); diff --git a/services/core/tests/integration/admin_session_archive_race_test.go b/services/core/tests/integration/admin_session_archive_race_test.go index e973a7c72..177dc8ecb 100644 --- a/services/core/tests/integration/admin_session_archive_race_test.go +++ b/services/core/tests/integration/admin_session_archive_race_test.go @@ -82,11 +82,3 @@ func TestManagedSessionArchiveOrdersConcurrentInput(t *testing.T) { } } } - -func TestManagedSessionArchiveRejectsFileManagedDeployment(t *testing.T) { - s, w, _ := managerFixture(t, 1, 1) - tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - if _, err := deploymentExecution(t, w).ArchiveSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 0); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("archive accepted file-managed deployment", err) - } -} diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index 00f830760..8e42cf4e7 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -41,7 +41,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} deployments := deploymentService(t, s) providerConfig := func(setup deployment.Setup) *execution.RuntimeProvider { - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} } configuration := execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { setup, err := deployments.Setup(ctx) diff --git a/services/core/tests/integration/credential_matrix_http_test.go b/services/core/tests/integration/credential_matrix_http_test.go index d424fc7d4..068423282 100644 --- a/services/core/tests/integration/credential_matrix_http_test.go +++ b/services/core/tests/integration/credential_matrix_http_test.go @@ -95,9 +95,9 @@ func TestCredentialNamespaceMatrix(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil }, func(_ context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil }) worker := startWorker(t, ctx, s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) var stop sync.Once diff --git a/services/core/tests/integration/device_bootstrap_binding_test.go b/services/core/tests/integration/device_bootstrap_binding_test.go index 597001774..bf28d963d 100644 --- a/services/core/tests/integration/device_bootstrap_binding_test.go +++ b/services/core/tests/integration/device_bootstrap_binding_test.go @@ -25,7 +25,7 @@ func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { t.Fatal(err) } onlineManagerNode(t, s, remote) - for _, nodeID := range []string{d.LocalNodeID, remote} { + for _, nodeID := range []string{d.NodeID, remote} { t.Run(nodeID, func(t *testing.T) { tenant, bearer := uuid.NewString(), uuid.NewString() session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), nodeID) diff --git a/services/core/tests/integration/hosted_initialization_failure_public_test.go b/services/core/tests/integration/hosted_initialization_failure_public_test.go index 543f11500..4364a0e8b 100644 --- a/services/core/tests/integration/hosted_initialization_failure_public_test.go +++ b/services/core/tests/integration/hosted_initialization_failure_public_test.go @@ -98,14 +98,17 @@ func (p *hostedFailureProvider) prepare(request proto.RuntimePreparePayload, _ [ return completedInitialization(request, nil) } -func hostedFailureStore(t *testing.T) *Store { +// hostedFailureStore returns a store whose Web deployment is claimed by the +// returned installation. +func hostedFailureStore(t *testing.T) (*Store, string) { t.Helper() _, pool := newManagedTestStore(t) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) if err != nil { t.Fatal(err) } - return NewWithCredentialCipher(pool, cipher) + s := NewWithCredentialCipher(pool, cipher) + return s, webDeployment(t, s, "e2b") } func hostedFailureSession(t *testing.T, s *Store, tenant string, input sessions.CreateSession) (sessions.Session, sessions.Environment) { @@ -126,10 +129,9 @@ func hostedFailureSession(t *testing.T, s *Store, tenant string, input sessions. return session, environment } -func failHostedInitialization(t *testing.T, s *Store, tenant string, environment sessions.Environment, p *hostedFailureProvider) { +func failHostedInitialization(t *testing.T, s *Store, key, tenant string, environment sessions.Environment, p *hostedFailureProvider) { t.Helper() - key := uuid.NewString() - w, _ := managedWorkerMode(t, s, key, p, false, true) + w, _ := managedWorkerMode(t, s, key, p, true) if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { t.Fatal(err) } @@ -180,12 +182,12 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { "Failed to provision environment: Skill installation failed", []string{"configure", "skill"}}, } { t.Run(test.name, func(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, test.input) p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: test.p.fail, skip: test.p.skip, result: test.p.result, err: test.p.err} - failHostedInitialization(t, s, tenant, environment, p) + failHostedInitialization(t, s, key, tenant, environment, p) if !reflect.DeepEqual(p.steps, test.steps) || p.kills != 0 || p.commandCalls.Load() != 0 { t.Fatal("failed initialization continued or reclaimed compute", p.steps, p.kills) } @@ -245,7 +247,7 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { // A pending initial input settles exactly as before; the one failed snapshot // carries both that settlement and the provisioning failure. func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "exit 3"}}}, @@ -253,7 +255,7 @@ func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { }) p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", result: failedInitialization(3)} - failHostedInitialization(t, s, tenant, environment, p) + failHostedInitialization(t, s, key, tenant, environment, p) read, err := sessionAdapter(s).GetSession(t.Context(), tenant, session.ID) if err != nil || read.PendingInput || read.EnvironmentInputActivity == nil || read.EnvironmentInputActivity.Status != "failed" || read.EnvironmentInputActivity.Failure != "environment_unavailable" || read.EnvironmentFailure == nil { @@ -279,20 +281,19 @@ func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { // stream ends after agent.session.failed; later input gets the observed 409; // delete succeeds; tenant B sees nothing; the canary never appears. func TestHostedInitializationFailurePublicHTTP(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 3"}}}, Metadata: map[string]string{"case": "setup-exit3"}, }) - key := uuid.NewString() // A failed typed Runtime receipt exposes only a safe status. p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", result: failedInitialization(3)} logs := &lockedBuffer{} previous := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) t.Cleanup(func() { slog.SetDefault(previous) }) - w, _ := managedWorkerMode(t, s, key, p, false, true) + w, _ := managedWorkerMode(t, s, key, p, true) auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: tenant, SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}, {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "tenant-b", TokenSHA256: runtimedevice.HashCredential(foreign), TenantID: uuid.NewString()}, diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index 383fde08d..f528d5ae0 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -10,26 +10,21 @@ import ( func (*lifecycleProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "NewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "GetCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Suspend": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Resume": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "KillCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } func (*lifecycleProvider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { @@ -62,46 +57,22 @@ func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sand func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } -func (*lifecycleProvider) ObserveBatch(context.Context, []runtimeobs.Target) ([]runtimeobs.BatchResult, error) { - return nil, &providercontract.UnsupportedError{Operation: "ObserveBatch", Reason: "fixture_operation_not_supported"} -} -func (*lifecycleProvider) DiscoverSelection(context.Context, sandbox.Selection) (sandbox.Selection, error) { - return sandbox.Selection{}, &providercontract.UnsupportedError{Operation: "DiscoverSelection", Reason: "fixture_operation_not_supported"} -} -func (*lifecycleProvider) VerifyCredential(context.Context, []sandbox.Reference) error { - return &providercontract.UnsupportedError{Operation: "VerifyCredential", Reason: "fixture_operation_not_supported"} -} func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations { return providercontract.Operations{ - "Create": {State: providercontract.Supported}, - "GetInfo": {State: providercontract.Supported}, - "Renew": {State: providercontract.Supported}, - "Kill": {State: providercontract.Supported}, - "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Supported}, - "NewCompute": {State: providercontract.Supported}, - "GetCompute": {State: providercontract.Supported}, - "Suspend": {State: providercontract.Supported}, - "Resume": {State: providercontract.Supported}, - "KillCompute": {State: providercontract.Supported}, - "DeleteSnapshot": {State: providercontract.Supported}, - "RunCommandCompute": {State: providercontract.Supported}, - "ResumeCompute": {State: providercontract.Supported}, - "ObservationProviderType": {State: providercontract.Supported}, - "ResolveObservationSource": {State: providercontract.Supported}, - "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "ObserveBatch": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "DiscoverSelection": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, - "VerifyCredential": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "Create": {State: providercontract.Supported}, + "GetInfo": {State: providercontract.Supported}, + "Renew": {State: providercontract.Supported}, + "Kill": {State: providercontract.Supported}, + "RunCommand": {State: providercontract.Supported}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteSnapshot": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, + "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, } } - -func (*lifecycleProvider) ObservationProviderType() string { return "fixture" } -func (p *lifecycleProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} - -func (*fakeCheckpointProvider) ObservationProviderType() string { return "fixture" } -func (p *fakeCheckpointProvider) ResolveObservationSource(context.Context) (runtimeobs.Source, error) { - return p, nil -} diff --git a/services/core/tests/integration/runtime_adoption_test.go b/services/core/tests/integration/runtime_adoption_test.go deleted file mode 100644 index 76ae44516..000000000 --- a/services/core/tests/integration/runtime_adoption_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package integration - -import ( - "fmt" - "reflect" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/google/uuid" -) - -func nodelessAllocationFixture(t *testing.T) (*Store, *Store, deployment.ProcessDeployment, deployment.Allocation) { - t.Helper() - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - d := deploymentSelection() - deploymentConfigure(t, w, &d) - tenant := uuid.NewString() - _, e := localEnvironment(t, s, tenant) - a, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: e.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) - if err != nil { - t.Fatal(err) - } - d.ProviderKind = "docker" - d.LocalNodeID = uuid.NewString() - d.LocalCredentialSHA256 = runtimedevice.HashCredential("node") - d.LocalMaxActive, d.LocalMaxRetained = 4, 16 - return s, w, d, a -} -func requireNoNodeBinding(t *testing.T, s *Store) { - t.Helper() - var nodes, placements, bound int - var kind string - if err := s.pool.QueryRow(t.Context(), `SELECT provider_kind,(SELECT count(*) FROM runtime_nodes),(SELECT count(*) FROM runtime_placements),(SELECT count(*) FROM runtime_allocations WHERE node_id IS NOT NULL) FROM runtime_deployment`).Scan(&kind, &nodes, &placements, &bound); err != nil { - t.Fatal(err) - } - if kind != "" || nodes != 0 || placements != 0 || bound != 0 { - t.Fatal("partial adoption", kind, nodes, placements, bound) - } -} -func TestHistoricalRuntimeResourcesCannotBeAdopted(t *testing.T) { - for _, state := range []string{"creating", "running", "cleanup_pending", "released"} { - for _, pending := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/pending=%t", state, pending), func(t *testing.T) { - s, w, d, a := nodelessAllocationFixture(t) - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET state=$2,create_settled=($2='released'),released_at=CASE WHEN $2='released' THEN clock_timestamp() ELSE NULL END WHERE id=$1`, a.ID, state) - if pending { - _, _ = localEnvironment(t, s, a.TenantID) - } - before, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.EnvironmentID}) - if err != nil { - t.Fatal(err) - } - err = deploymentExecution(t, w).ConfigureProcess(t.Context(), &d) - if state == "released" && !pending { - if err != nil { - t.Fatal("released history blocked fresh configuration", err) - } - } else { - if err == nil { - t.Fatal("historical resources adopted") - } - requireNoNodeBinding(t, s) - } - after, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.EnvironmentID}) - if err != nil || !reflect.DeepEqual(before, after) { - t.Fatal("retained receipt changed", err) - } - }) - } - } -} diff --git a/services/core/tests/integration/runtime_capabilities_pending_test.go b/services/core/tests/integration/runtime_capabilities_pending_test.go index 5bbe67cf6..3e3c01d2d 100644 --- a/services/core/tests/integration/runtime_capabilities_pending_test.go +++ b/services/core/tests/integration/runtime_capabilities_pending_test.go @@ -21,6 +21,7 @@ func TestManagedCapabilitiesWaitBeforeInitializationClaim(t *testing.T) { t.Fatal(err) } s := NewWithCredentialCipher(pool, cipher) + key := webDeployment(t, s, "e2b") tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{ Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), @@ -35,8 +36,7 @@ func TestManagedCapabilitiesWaitBeforeInitializationClaim(t *testing.T) { t.Fatal(err) } provider := &initializingProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, initializationPeer: initializationPeer{deferred: true}} - key := uuid.NewString() - worker, _ := managedWorkerMode(t, s, key, provider, false, true) + worker, _ := managedWorkerMode(t, s, key, provider, true) owner, err := worker.ProvisionEnvironment(t.Context(), tenant, env.ID, key) if err != nil || initializationState(t, s, owner.TenantID, owner.EnvironmentID) != "pending" { t.Fatal(owner, err) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index c6949a6de..73f5e29d6 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -17,7 +17,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -261,6 +260,7 @@ type computeLifecycleFixture struct { worker *execution.Worker stop func() key string + node string policy execution.RuntimeSuspensionPolicy } @@ -280,28 +280,21 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: uuid.NewString(), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + view, err := deploymentService(t, s).View(t.Context()) + if err != nil { + t.Fatal(err) + } + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID f.start() return f } func (f *computeLifecycleFixture) start() { t := f.t t.Helper() - dispatcher := &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: f.provider, Suspension: &f.policy}} - // Closing the previous Worker's connection can return before PostgreSQL drops its advisory lock. - deadline := time.Now().Add(2 * time.Second) - var w *execution.Worker - var err error - for { - w, err = startWorkerErr(t.Context(), f.store, dispatcher) - if err == nil || !errors.Is(err, pgunit.ErrLeaseHeld) || !time.Now().Before(deadline) { - break - } - time.Sleep(20 * time.Millisecond) - } - if err != nil { - t.Fatal(err) - } + w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) + // The Worker's claim starts a new owner epoch, in which the node reconnects. + onlineManagerNode(t, f.store, f.node) var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -521,47 +514,3 @@ func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) }) } } - -func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { - f := newComputeLifecycleFixture(t, 1, 2) - tenant, _, env, owner := f.create() - tenant2, _, env2 := managedSession(t, f.store) - if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatalf("active capacity ignored: %v", err) - } - if f.provider.creates != 1 { - t.Fatal("capacity rejection allocated compute") - } - f.complete(owner) - f.phase(tenant, env.ID, "suspended") - second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) - if err != nil { - t.Fatal(err) - } - pending := f.queued(owner) - for range 4 { - f.worker.ReconcileManagedRuntimes(t.Context()) - } - first, err := deploymentStore(f.store).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) - if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { - t.Fatal("wake exceeded active capacity", err) - } - f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) - f.provider.mu.Lock() - b := f.provider.bootstraps[second.ID] - f.provider.mu.Unlock() - if err := f.provider.connect(t.Context(), b); err != nil { - t.Fatal(err) - } - second = f.phase(tenant2, env2.ID, "running") - f.complete(second) - f.phase(tenant2, env2.ID, "suspended") - // Both retained allocations count even when their source VMs are gone. - tenant3, _, env3 := managedSession(t, f.store) - if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatalf("retained capacity ignored: %v", err) - } - if f.provider.creates != 2 { - t.Fatal("retained limit created a third allocation") - } -} diff --git a/services/core/tests/integration/runtime_configuration_cleanup_test.go b/services/core/tests/integration/runtime_configuration_cleanup_test.go index 2196f7674..c08993908 100644 --- a/services/core/tests/integration/runtime_configuration_cleanup_test.go +++ b/services/core/tests/integration/runtime_configuration_cleanup_test.go @@ -84,7 +84,7 @@ func TestManagedRuntimeConfigurationCleanup(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &configurationCleanupProvider{ lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: test.loseCreate}, rejectCreate: test.rejectCreate, settleCreate: test.settleCreate, diff --git a/services/core/tests/integration/runtime_connection_test.go b/services/core/tests/integration/runtime_connection_test.go index 19947a340..7bc66fb03 100644 --- a/services/core/tests/integration/runtime_connection_test.go +++ b/services/core/tests/integration/runtime_connection_test.go @@ -22,6 +22,7 @@ import ( func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, environment := managedSession(t, s) server := httptest.NewUnstartedServer(nil) wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" @@ -33,11 +34,7 @@ func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { server.Start() t.Cleanup(func() { server.Close(); runtime.CloseConnections(registry) }) p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - key := uuid.NewString() - start := func() *execution.Worker { - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: server.URL + "/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) - return w - } + start := func() *execution.Worker { return startWebWorker(t, s, registry, key, p, nil) } stop := func(w *execution.Worker) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/runtime_creation_settlement_test.go b/services/core/tests/integration/runtime_creation_settlement_test.go index b827e9c49..54390356d 100644 --- a/services/core/tests/integration/runtime_creation_settlement_test.go +++ b/services/core/tests/integration/runtime_creation_settlement_test.go @@ -42,7 +42,7 @@ func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { for _, cancelled := range []bool{false, true} { t.Run(map[bool]string{false: "live caller", true: "cancelled caller"}[cancelled], func(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) @@ -80,7 +80,7 @@ func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { } func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{foreign: true} w, _ := managedWorker(t, s, key, p) tenant, _, environment := managedSession(t, s) @@ -94,7 +94,7 @@ func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { } func TestManagedRuntimeObservedSettlementAllowsOwnedCleanup(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{observeSettled: true} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) diff --git a/services/core/tests/integration/runtime_deployment_test.go b/services/core/tests/integration/runtime_deployment_test.go index 28d6bb24c..49cd6e6ac 100644 --- a/services/core/tests/integration/runtime_deployment_test.go +++ b/services/core/tests/integration/runtime_deployment_test.go @@ -1,12 +1,9 @@ package integration import ( - "context" "encoding/json" "errors" - "strings" "testing" - "time" "github.com/google/uuid" @@ -16,290 +13,94 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func deploymentSelection() deployment.ProcessDeployment { - return deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} -} - -func deploymentConfigure(t *testing.T, w *Store, config *deployment.ProcessDeployment) { - t.Helper() - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), config); err != nil { - t.Fatal(err) - } - if config != nil && config.ProviderKind != "" { - legacyRuntimeSpecification(t, w, config.ProviderKind) - } -} - -// Lower-level legacy fixtures supply verified metadata without replaying the -// configuration transition being tested. Web setup owns this write in production. -func legacyRuntimeSpecification(t *testing.T, w *Store, provider string) { - t.Helper() - spec := SandboxDeploymentTestSpec(provider) - raw, _ := json.Marshal(spec) - if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification=$1", raw); err != nil { - t.Fatal(err) - } - if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_nodes SET deployment_generation=1,ready_generation=1,specification_digest=$1", spec.Digest(provider)); err != nil { - t.Fatal(err) - } -} - -func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { +// An installation never adopts hosted work admitted before it claimed the +// deployment: a pending Session or an unreleased allocation refuses the claim. +func TestRuntimeDeploymentClaimAdoptsNoUnclaimedWork(t *testing.T) { s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - tenant := uuid.NewString() - session, _ := localEnvironment(t, s, tenant) - old := deploymentSelection() - // First selection is allowed for work that has never had an installation. - deploymentConfigure(t, w, &old) - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { - t.Fatal("pending Session migrated", err) + changes := deploymentExecution(t, executionWriter(t, s)) + installation, tenant := uuid.NewString(), uuid.NewString() + pending, _ := localEnvironment(t, s, tenant) + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("pending Session adopted", err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { - t.Fatal("pending Session orphaned by removing provider") - } - if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { + if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: pending.ID}); err != nil { t.Fatal(err) } - deploymentConfigure(t, w, &next) -} - -func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) { - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - tenant := uuid.NewString() session, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) + owner, err := changes.ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil || !strings.Contains(err.Error(), "no verified backend identity") { - t.Fatal("legacy allocation silently adopted", err) + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("unclaimed allocation adopted", err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).RequestCleanup(t.Context(), owner); err != nil { + if _, err := changes.RequestCleanup(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := changes.ReleaseAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("unknown creation lost cleanup ownership", err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil { - t.Fatal("deleted unknown allocation did not block adoption") + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("deleted unknown allocation did not block adoption", err) } - if _, err := deploymentExecution(t, w).SettleCreation(t.Context(), owner); err != nil { + if _, err := changes.SettleCreation(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); err != nil { + if _, err := changes.ReleaseAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - deploymentConfigure(t, w, &old) - _, environment = localEnvironment(t, s, tenant) - owner, err = deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("unknown creation did not block switch", err) - } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { - t.Fatal("removing adapter orphaned unknown creation") - } - replay, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil || !replay.Replayed || replay.ID != owner.ID { - t.Fatal("maintenance blocked receipt replay", replay, err) - } - if _, err := deploymentExecution(t, w).RequestCleanup(t.Context(), owner); err != nil { - t.Fatal("maintenance blocked cleanup", err) + if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("owner without runtimes accepted a claimed deployment", err) } } -func TestRuntimeDeploymentMaintenancePreservesCreationRetriesAndOtherPlacements(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - deploymentConfigure(t, w, &old) +func TestRuntimeDeploymentResetPreservesCreationRetriesAndOtherPlacements(t *testing.T) { + s, w, installation := managedArchiveFixture(t) tenant := uuid.NewString() input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`)} existing, err := s.CreateSession(t.Context(), tenant, input) if err != nil { t.Fatal(err) } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) + ctx := SandboxResetTestContext(t.Context()) + if _, err := startReset(t, ctx, w, installation, deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } replay, err := s.CreateSession(t.Context(), tenant, input) if err != nil || replay.ID != existing.ID { t.Fatal("creation retry lost identity", err) } input.IdempotencyKey = uuid.NewString() - if _, err := s.CreateSession(t.Context(), tenant, input); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance created hosted Session", err) + if _, err := s.CreateSession(t.Context(), tenant, input); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset created hosted Session", err) } var count int - if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 1 { + if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 1 { t.Fatal("rejection left partial Session", count, err) } - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance reserved new allocation", err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset reserved new allocation", err) } for _, kind := range []string{"none", "self_hosted"} { input.IdempotencyKey = uuid.NewString() input.Configuration = json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"` + kind + `"}}`) if _, err := s.CreateSession(t.Context(), tenant, input); err != nil { - t.Fatal("maintenance blocked unrelated placement", kind, err) + t.Fatal("reset blocked unrelated placement", kind, err) } } - old.AdmissionPaused = false - deploymentConfigure(t, w, &old) - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("wrong installation reserved resource", err) - } - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())); err != nil { - t.Fatal("resume did not reopen allocation", err) - } -} - -func TestRuntimeDeploymentMaintenanceSerializesHostedCreation(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - config := deploymentSelection() - deploymentConfigure(t, w, &config) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - tx, err := pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(context.Background()) - var blocker int32 - if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - tenant := uuid.NewString() - go func() { - _, err := s.CreateSession(ctx, tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)}) - done <- err - }() - runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) - if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET admission_paused=true"); err != nil { - t.Fatal(err) - } - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - if err := <-done; !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("creation bypassed committed maintenance", err) - } - var count int - if err := pool.QueryRow(ctx, "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 0 { - t.Fatal("racing creation left partial work", count, err) - } -} - -func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing.T) { - for _, state := range []string{"running", "suspended", "cleanup_pending"} { - t.Run(state, func(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - deploymentConfigure(t, w, &old) - tenant := uuid.NewString() - _, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - if state == "suspended" { - _, err = pool.Exec(t.Context(), `UPDATE runtime_allocations SET state='running',create_settled=true,compute_phase='suspended',compute_state='{"snapshot":{"id":"retained-test-snapshot"}}' WHERE id=$1`, owner.ID) - } else { - _, err = pool.Exec(t.Context(), "UPDATE runtime_allocations SET state=$2,create_settled=true WHERE id=$1", owner.ID, state) - } - if err != nil { - t.Fatal(err) - } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - var before, after, oldIdentity, newIdentity string - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&before); err != nil { - t.Fatal(err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&oldIdentity); err != nil { - t.Fatal(err) - } - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("retained resource allowed switch", state, err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&after); err != nil { - t.Fatal(err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&newIdentity); err != nil { - t.Fatal(err) - } - if before != after || oldIdentity != newIdentity { - t.Fatal("refused switch changed existing ownership") - } - }) - } -} - -func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - config := deploymentSelection() - deploymentConfigure(t, w, &config) - tenant := uuid.NewString() - _, environment := localEnvironment(t, s, tenant) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - tx, err := pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(context.Background()) - var blocker int32 - if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { + if err := deploymentExecution(t, w).CancelReset(ctx, installation, 1); err != nil { t.Fatal(err) } - allocated := make(chan error, 1) - go func() { - _, err := deploymentExecution(t, w).ReserveAllocation(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, config.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - allocated <- err - }() - runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, allocated) - maintaining := make(chan error, 1) - maintenance := config - maintenance.AdmissionPaused = true - changes := deploymentExecution(t, w) - go func() { maintaining <- changes.ConfigureProcess(ctx, &maintenance) }() - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - if err := <-allocated; err != nil { - t.Fatal("earlier allocation lost ownership", err) - } - if err := <-maintaining; err != nil { - t.Fatal(err) - } - owner, err := deploymentStore(w).EnvironmentAllocation(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}) - if err != nil || owner.State != "creating" || owner.CreateSettled { - t.Fatal("maintenance changed uncertain receipt", owner, err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { + t.Fatal("wrong installation reserved resource", err) } - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(ctx, &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("earlier in-flight allocation omitted from switch guard", err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())); err != nil { + t.Fatal("cancelled reset did not reopen allocation", err) } } diff --git a/services/core/tests/integration/runtime_deployment_worker_test.go b/services/core/tests/integration/runtime_deployment_worker_test.go index 42ce2fae8..6e852099f 100644 --- a/services/core/tests/integration/runtime_deployment_worker_test.go +++ b/services/core/tests/integration/runtime_deployment_worker_test.go @@ -1,7 +1,7 @@ package integration import ( - "strings" + "errors" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -13,7 +13,7 @@ import ( func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") old := &lifecycleProvider{resources: map[string]sandbox.Info{}} worker, stop := managedWorker(t, s, key, old) tenant, _, environment := managedSession(t, s) @@ -23,25 +23,15 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) } stop() replacement := &lifecycleProvider{resources: map[string]sandbox.Info{}} - config := &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64), Provider: replacement, AdmissionPaused: true} - start := func(config *execution.RuntimeProvider) error { - _, err := startWorkerErr(t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: config}) + start := func(runtimes *execution.RuntimeProvider) error { + _, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) return err } - if err := start(config); err == nil || !strings.Contains(err.Error(), "maintenance") { - t.Fatal("startup switched active deployment", err) + if err := start(webRuntimes(t, s, uuid.NewString(), replacement, nil)); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("startup switched the claimed installation", err) } - worker, stop = managedWorkerMode(t, s, key, old, true) - replay, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) - if err != nil || !replay.Replayed || replay.ID != owner.ID { - t.Fatal("maintenance interrupted existing allocation", replay, err) - } - stop() - if err := start(config); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { - t.Fatal("startup abandoned retained allocation", err) - } - if err := start(nil); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { - t.Fatal("omitted configuration abandoned deployment", err) + if err := start(nil); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("startup without runtimes abandoned the claimed deployment", err) } if replacement.creates != 0 || replacement.kills != 0 { t.Fatal("rejected startup touched new backend") @@ -51,6 +41,10 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) t.Fatal("rejected startup rewrote resource owner", got, err) } // Failed startup relinquishes its lease, so the original backend can resume. - _, stop = managedWorker(t, s, key, old) + worker, stop = managedWorker(t, s, key, old) + replay, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) + if err != nil || !replay.Replayed || replay.ID != owner.ID { + t.Fatal("rejected startup interrupted the existing allocation", replay, err) + } stop() } diff --git a/services/core/tests/integration/runtime_initialization_test.go b/services/core/tests/integration/runtime_initialization_test.go index 57a4bf5cc..8acc75925 100644 --- a/services/core/tests/integration/runtime_initialization_test.go +++ b/services/core/tests/integration/runtime_initialization_test.go @@ -52,6 +52,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { t.Fatal(err) } s := NewWithCredentialCipher(pool, cipher) + key := webDeployment(t, s, "e2b") tenant := uuid.NewString() input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`), InitialFiles: []environmentconfig.InitialFile{{Type: "inline", Path: "/workspace/a", Data: []byte("first")}, {Type: "inline", Path: "/workspace/b", Data: []byte("second")}}} if setupOnly { @@ -85,8 +86,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } return completedInitialization(proto.RuntimePreparePayload{}, nil) } - key := uuid.NewString() - w, stop := managedWorkerMode(t, s, key, p, false, true) + w, stop := managedWorkerMode(t, s, key, p, true) if mode == "restart" { awaitInitialization(t, s, tenant, env.ID, "failed") if p.writes.Load() != 0 { @@ -122,7 +122,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { t.Fatal("completed preparation blocked", err) } stop() - _, _ = managedWorkerMode(t, s, key, p, false, true) + _, _ = managedWorkerMode(t, s, key, p, true) time.Sleep(350 * time.Millisecond) if int(p.writes.Load()) != expectedSteps { t.Fatal("completed preparation replayed") @@ -141,7 +141,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } func TestManagedRuntimePreparationAllOperationsUsePeer(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) var archive bytes.Buffer writer := zip.NewWriter(&archive) for path, body := range map[string]string{"proof/.codex-plugin/plugin.json": `{"name":"plugin","description":"A plugin.","skills":"./skills"}`, "proof/skills/example/SKILL.md": "---\nname: plugin-proof\ndescription: A plugin Skill.\n---\nProof."} { @@ -181,8 +181,7 @@ func TestManagedRuntimePreparationAllOperationsUsePeer(t *testing.T) { actionsMu.Unlock() return completedInitialization(request, data) } - key := uuid.NewString() - worker, _ := managedWorkerMode(t, s, key, provider, false, true) + worker, _ := managedWorkerMode(t, s, key, provider, true) if _, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_input_admission_test.go b/services/core/tests/integration/runtime_input_admission_test.go index 8c45dac30..725bb0567 100644 --- a/services/core/tests/integration/runtime_input_admission_test.go +++ b/services/core/tests/integration/runtime_input_admission_test.go @@ -7,13 +7,15 @@ import ( "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { +func TestManagedRuntimeResetPreservesCancelAndRetry(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, _ := managedSession(t, s) inputs := []sessions.Input{messageInput("accepted work")} accepted, err := submitInputs(t.Context(), s, tenant, session.ID, "work", inputs) @@ -21,10 +23,13 @@ func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { t.Fatal(err) } p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - w, stop := managedWorkerMode(t, s, uuid.NewString(), p, true) + w, stop := managedWorker(t, s, key, p) defer stop() - if _, err := w.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance accepted new hosted Session", err) + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + if _, err := w.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset accepted new hosted Session", err) } cancel := []sessions.Input{{Kind: "cancel", Payload: json.RawMessage(`{}`)}} first, err := w.SubmitInputs(t.Context(), tenant, session.ID, "cancel", cancel) @@ -40,7 +45,7 @@ func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { t.Fatal("matching input retry lost its accepted outcome", retry, err) } if _, err := w.SubmitInputs(t.Context(), uuid.NewString(), session.ID, "cancel", cancel); !errors.Is(err, sessions.ErrNotFound) { - t.Fatal("maintenance weakened tenant isolation", err) + t.Fatal("reset weakened tenant isolation", err) } if p.creates != 0 { t.Fatal("existing controls provisioned a new Runtime") diff --git a/services/core/tests/integration/runtime_lifecycle_nodes_test.go b/services/core/tests/integration/runtime_lifecycle_nodes_test.go index a02c00452..72f26498a 100644 --- a/services/core/tests/integration/runtime_lifecycle_nodes_test.go +++ b/services/core/tests/integration/runtime_lifecycle_nodes_test.go @@ -37,7 +37,7 @@ func lifecycleTestSession(t *testing.T, s *Store, node string) (string, sessions } return tenant, session } -func lifecycleTestAllocation(t *testing.T, s, w *Store, d deployment.ProcessDeployment, node string) deployment.Allocation { +func lifecycleTestAllocation(t *testing.T, s, w *Store, d managerNode, node string) deployment.Allocation { t.Helper() tenant, session := lifecycleTestSession(t, s, node) allocation, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential(uuid.NewString())) @@ -52,20 +52,20 @@ func TestRuntimeLifecycleNodePagesAreIndependent(t *testing.T) { other := lifecycleTestNode(t, s) var allocated, pending []string for range 34 { - allocated = append(allocated, lifecycleTestAllocation(t, s, w, d, d.LocalNodeID).ID) - _, session := lifecycleTestSession(t, s, d.LocalNodeID) + allocated = append(allocated, lifecycleTestAllocation(t, s, w, d, d.NodeID).ID) + _, session := lifecycleTestSession(t, s, d.NodeID) pending = append(pending, session.Environment.ID) } second := lifecycleTestAllocation(t, s, w, d, other) _, secondPending := lifecycleTestSession(t, s, other) // Offline and unresolved cleanup remain discoverable without changing placement. - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET state='cleanup_pending' WHERE node_id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET state='cleanup_pending' WHERE node_id=$1", d.NodeID); err != nil { t.Fatal(err) } - for _, node := range []string{d.LocalNodeID, other} { + for _, node := range []string{d.NodeID, other} { var gotAlloc, gotPending []string cursor := "" for range 4 { @@ -181,7 +181,7 @@ func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { t.Fatal(err) } nodes, err = deploymentStore(w).LifecycleNodes(t.Context()) - if err != nil || len(nodes) != 1 || nodes[0] != d.LocalNodeID { + if err != nil || len(nodes) != 1 || nodes[0] != d.NodeID { t.Fatal("removed node discovered", nodes, err) } } @@ -190,14 +190,14 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { for _, mutation := range []string{"DELETE FROM runtime_placements WHERE environment_id=$1", "UPDATE runtime_placements SET released_at=clock_timestamp() WHERE environment_id=$1"} { t.Run(mutation[:6], func(t *testing.T) { s, w, d := managerFixture(t, 4, 4) - tenant, session := lifecycleTestSession(t, s, d.LocalNodeID) + tenant, session := lifecycleTestSession(t, s, d.NodeID) if _, err := s.pool.Exec(t.Context(), mutation, session.Environment.ID); err != nil { t.Fatal(err) } if _, err := deploymentService(t, w).LifecycleNode(t.Context(), tenant, session.Environment.ID); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("invalid placement routed", err) } - rows, err := deploymentStore(w).UnallocatedEnvironments(t.Context(), d.LocalNodeID, "") + rows, err := deploymentStore(w).UnallocatedEnvironments(t.Context(), d.NodeID, "") if err != nil || len(rows) != 0 { t.Fatal("invalid placement provisioned", rows, err) } @@ -206,7 +206,12 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { } func TestRuntimeLifecycleNodelessLane(t *testing.T) { - s, w, _, a := nodelessAllocationFixture(t) + s, w, installation := managedArchiveFixture(t) + _, reserved := localEnvironment(t, s, uuid.NewString()) + a, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: reserved.TenantID, EnvironmentID: reserved.ID}, installation, runtimedevice.HashCredential("runtime")) + if err != nil { + t.Fatal(err) + } nodes, err := deploymentStore(w).LifecycleNodes(t.Context()) if err != nil || !slices.Equal(nodes, []string{""}) { t.Fatal(nodes, err) diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index f03f85e0a..8da737e4a 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -72,12 +72,15 @@ func (p *lifecycleProvider) RunCommand(context.Context, sandbox.Reference, sandb return sandbox.CommandResult{}, errors.New("not used") } +// managedWorker starts a Worker that runs the Web setup webDeployment +// committed for installation key on p. func managedWorker(t *testing.T, s *Store, key string, p sandbox.SandboxProvider) (*execution.Worker, func()) { t.Helper() return managedWorkerMode(t, s, key, p, false) } -func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProvider, maintenance bool, run ...bool) (*execution.Worker, func()) { +// managedWorkerMode is managedWorker that also runs the Worker when run is set. +func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProvider, run bool) (*execution.Worker, func()) { t.Helper() registry := runtimegateway.NewRegistry() if peer, ok := p.(interface { @@ -88,8 +91,8 @@ func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProv t.Cleanup(server.Close) peer.setRuntimeGateway(t, "ws"+strings.TrimPrefix(server.URL, "http"), registry) } - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p, AdmissionPaused: maintenance}}) - if len(run) > 0 && run[0] { + w := startWebWorker(t, s, registry, key, p, nil) + if run { ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { done <- w.Run(ctx) }() @@ -146,8 +149,8 @@ func reconcileManagedState(t *testing.T, w *execution.Worker, s *Store, tenant, func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true} w, stop := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) @@ -185,8 +188,8 @@ func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true, absent: true} w, _ := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) @@ -213,37 +216,10 @@ func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { } } -func TestManagedRuntimeExpiryRevokesWhenProviderUnavailable(t *testing.T) { - s, _ := newManagedTestStore(t) - tenant, _, env := managedSession(t, s) - key := uuid.NewString() - p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - w, _ := managedWorker(t, s, key, p) - owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) - if err != nil { - t.Fatal(err) - } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '61 minutes' WHERE id=$1", owner.ID); err != nil { - t.Fatal(err) - } - p.unavailable = true - reconcileManagedState(t, w, s, tenant, env.ID, "cleanup_pending") - got, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) - if err != nil || got.State != "cleanup_pending" { - t.Fatalf("expiry lost on provider failure: %+v %v", got, err) - } - if _, ok, err := sessionAdapter(s).GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { - t.Fatal("expired credential still authenticates") - } - if p.kills != 0 { - t.Fatal("unavailable provider misreported cleanup") - } -} - func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, _, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} w, _ := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index d581b19f6..771c4c99f 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -18,7 +18,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -124,21 +123,13 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { cp.mu.Unlock() server.Close() }) - f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: uuid.NewString(), nodeA: uuid.NewString(), nodeB: uuid.NewString()} + f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: webDeployment(t, s, "microsandbox"), nodeA: uuid.NewString(), nodeB: uuid.NewString()} // Keep restored compute awake throughout the isolation assertions. // The suspension setup explicitly dates its activity two minutes in the past. - policy := &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour, MaxActive: 100, MaxRetained: 100} - f.worker = startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: p, ProviderKind: "microsandbox", LocalNodeID: f.nodeA, LocalCredentialSHA256: runtimedevice.HashCredential("local-credential"), LocalMaxActive: 100, LocalMaxRetained: 100, Suspension: policy}}) - spec := SandboxDeploymentTestSpec("microsandbox") - raw, _ := json.Marshal(spec) - if _, err := pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification=$1", raw); err != nil { - t.Fatal(err) - } - if _, err := pool.Exec(t.Context(), "UPDATE runtime_nodes SET specification_digest=$1,deployment_generation=1", spec.Digest("microsandbox")); err != nil { - t.Fatal(err) - } + f.worker = startWebWorker(t, s, registry, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour}) t.Cleanup(f.stop) f.epoch = fixtureOwnerEpoch(t, s) + f.enroll(f.nodeA) f.enroll(f.nodeB) f.online(f.nodeA) f.online(f.nodeB) diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index cf11c0a42..c00d88ceb 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -17,19 +17,15 @@ import ( "github.com/jackc/pgx/v5" ) -func managerFixture(t *testing.T, active, retained int) (*Store, *Store, deployment.ProcessDeployment) { +// managerNode is the deployment managerFixture sets up: installation +// InstallationID runs Docker on the one online node NodeID. +type managerNode struct{ InstallationID, NodeID string } + +func managerFixture(t *testing.T, active, retained int) (*Store, *Store, managerNode) { t.Helper() - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - d := deploymentSelection() - d.ProviderKind = "docker" - d.LocalNodeID = uuid.NewString() - d.LocalCredentialSHA256 = runtimedevice.HashCredential("local-node-credential") - d.LocalMaxActive = active - d.LocalMaxRetained = retained - deploymentConfigure(t, w, &d) - onlineManagerNode(t, s, d.LocalNodeID) - return s, w, d + s, w, view, _ := webSpecificationFixture(t, "docker") + node := enrollNode(t, s, view, deployment.Capacity{MaxActive: active, MaxRetained: retained}) + return s, w, managerNode{InstallationID: view.InstallationID, NodeID: node.NodeID} } func onlineManagerNode(t *testing.T, s *Store, id string) string { t.Helper() @@ -158,7 +154,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { if err != nil || len(nodes) != 1 || nodes[0].Active != 1 || nodes[0].Retained != 1 || nodes[0].Reserved != 1 { t.Fatal(nodes, err) } - if err := service.RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := service.RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed pending placement", err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: retained.ID}); err != nil { @@ -169,7 +165,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } replay, err := s.CreateSession(t.Context(), tenant, input) @@ -180,7 +176,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { t.Fatal("foreign placement leaked", err) } placement, err := sessionRuntimePlacement(t.Context(), s, tenant, first.ID) - if err != nil || placement.NodeID != d.LocalNodeID || placement.Available { + if err != nil || placement.NodeID != d.NodeID || placement.Available { t.Fatal(placement, err) } } @@ -220,7 +216,9 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { } } epoch := managerEpoch(t, s) - deploymentConfigure(t, w, &d) + if err := deploymentExecution(t, w).Claim(t.Context(), d.InstallationID); err != nil { + t.Fatal(err) + } if next := managerEpoch(t, s); next != epoch+1 { t.Fatal(next) } @@ -230,7 +228,7 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("stale node admitted", err) } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) if err := nodes.RemoveNode(t.Context(), input.NodeID); err != nil { t.Fatal(err) } @@ -254,7 +252,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal(err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: pending.ID}); err != nil { @@ -267,7 +265,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("unknown cleanup released node", err) } retained, err = deploymentExecution(t, w).SettleCreation(t.Context(), retained) @@ -277,22 +275,8 @@ func TestRuntimeNodesRetention(t *testing.T) { if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), retained); err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrLocalNodeConfigured) { - t.Fatal("configured local node was removed", err) - } - if _, err := nodes.AuthenticateNode(t.Context(), next.LocalNodeID, "local-node-credential"); err != nil { - t.Fatal("rejected removal changed local credentials", err) - } - next.AdmissionPaused = true - deploymentConfigure(t, w, &next) - detached := next - detached.LocalNodeID = "" - detached.LocalCredentialSHA256 = "" - detached.LocalMaxActive, detached.LocalMaxRetained = 0, 0 - detached.BackendFingerprint = strings.Repeat("b", 64) - deploymentConfigure(t, w, &detached) - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); err != nil { - t.Fatal("detached resolved node cannot be removed", err) + if err := nodes.RemoveNode(t.Context(), next.NodeID); err != nil { + t.Fatal("released node cannot be removed", err) } } func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { @@ -371,22 +355,17 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 days' WHERE id=$1", owner.ID); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } offline, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || offline.Expired || offline.State != "running" { t.Fatal("offline treated as destructive expiry", offline, err) } - if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("offline ownership discarded", err) } - changed := d - changed.LocalNodeID = uuid.NewString() - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &changed); err == nil { - t.Fatal("lost local state created replacement identity") - } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) resumed, err := deploymentExecution(t, w).ObserveRunning(t.Context(), offline) if err != nil || resumed.ID != owner.ID || resumed.DeviceID != owner.DeviceID || resumed.NodeID != owner.NodeID { t.Fatal("reconnect changed instance", resumed, err) @@ -397,20 +376,20 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2::jsonb,compute_retained_until=clock_timestamp()+interval '1 day' WHERE id=$1", owner.ID, json.RawMessage(`{"snapshot":{"id":"same-snapshot"}}`)); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } retained, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || retained.Expired || string(retained.ComputeState) != `{"snapshot": {"id": "same-snapshot"}}` { t.Fatal(retained, err) } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) same, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || same.ID != owner.ID || string(same.ComputeState) != string(retained.ComputeState) { t.Fatal("snapshot changed across reconnect", same, err) } placement, err := sessionRuntimePlacement(t.Context(), s, tenant, session.ID) - if err != nil || placement.NodeID != d.LocalNodeID { + if err != nil || placement.NodeID != d.NodeID { t.Fatal(placement, err) } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1", owner.ID); err != nil { diff --git a/services/core/tests/integration/runtime_observation_test.go b/services/core/tests/integration/runtime_observation_test.go index 0c178075c..5ad2acf2b 100644 --- a/services/core/tests/integration/runtime_observation_test.go +++ b/services/core/tests/integration/runtime_observation_test.go @@ -31,7 +31,7 @@ func TestRuntimeNodeObservationRetainsResourcesAndFencesStaleResults(t *testing. if err != nil || retained.State != "running" || retained.ID != owner.ID || retained.ObservationError != "node_unavailable" { t.Fatal(retained, err) } - if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("diagnostic released resource", err) } // A new lifecycle observation must not be erased by an earlier result. diff --git a/services/core/tests/integration/runtime_pending_test.go b/services/core/tests/integration/runtime_pending_test.go index 8245750f5..06af06ef7 100644 --- a/services/core/tests/integration/runtime_pending_test.go +++ b/services/core/tests/integration/runtime_pending_test.go @@ -15,6 +15,7 @@ import ( func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, idle, idleEnvironment := managedSession(t, s) initial, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`), InitialInputs: []sessions.Input{messageInput("hello")}}) if err != nil { @@ -24,12 +25,8 @@ func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: deleted.TenantID, SessionID: deleted.ID}); err != nil { t.Fatal(err) } - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - start := func() *execution.Worker { - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) - return w - } + start := func() *execution.Worker { return startWebWorker(t, s, runtimegateway.NewRegistry(), key, p, nil) } stop := func(w *execution.Worker) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/runtime_scan_test.go b/services/core/tests/integration/runtime_scan_test.go index c5221dbc1..5f88de644 100644 --- a/services/core/tests/integration/runtime_scan_test.go +++ b/services/core/tests/integration/runtime_scan_test.go @@ -8,8 +8,6 @@ import ( "testing" "time" - "github.com/google/uuid" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -30,7 +28,7 @@ func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { t.Run(fmt.Sprint(count), func(t *testing.T) { s, _ := newManagedTestStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := uuid.NewString() + key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) var ids []string for range count { @@ -69,7 +67,7 @@ func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { func TestManagedRuntimeScanEmptyAfterCleanupAndCanceledCall(t *testing.T) { s, _ := newManagedTestStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := uuid.NewString() + key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) tenant, session, env := managedSession(t, s) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 402338582..6ca4cd340 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -261,47 +261,6 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } } -func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { - s, pool := testStore(t) - w := executionWriter(t, s) - provider := uuid.NewString() - cases := []struct { - state, phase string - count bool - }{ - {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, - } - want := int64(0) - wantRetained := int64(0) - for _, item := range cases { - tenant := uuid.NewString() - _, env := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) - if item.count { - want++ - } - if item.state != "released" { - wantRetained++ - } - retained, err := deploymentStore(w).CountRetainedAllocations(t.Context(), provider) - if err != nil || retained != wantRetained { - t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) - } - got, err := deploymentStore(w).CountComputeReservations(t.Context(), provider) - if err != nil || got != want { - t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) - } - } - got, err := deploymentStore(w).CountComputeReservations(t.Context(), uuid.NewString()) - if err != nil || got != 0 { - t.Fatal("capacity crossed installation boundary", got, err) - } -} - func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { _, w, pool, owner := runtimeSuspensionFixture(t) turn := runtimeSuspensionCompleted(t, pool, owner) @@ -457,7 +416,7 @@ func TestRuntimeComputePhaseChangedAtInNodeAllocations(t *testing.T) { } listed := func() deployment.NodeAllocation { t.Helper() - items, err := deploymentStore(s).NodeAllocations(t.Context(), d.LocalNodeID) + items, err := deploymentStore(s).NodeAllocations(t.Context(), d.NodeID) if err != nil || len(items) != 1 || items[0].ID != allocation.ID { t.Fatal(items, err) } diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index 0d29a6720..aedeca68c 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -73,14 +73,8 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } - worker := startWorker(t, t.Context(), f.store, &execution.Dispatcher{ - Registry: f.provider.registry, - ManagedRuntimes: &execution.RuntimeProvider{ - CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, - BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Provider: provider, Suspension: &f.policy, - }, - }) + worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) + onlineManagerNode(t, f.store, f.node) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) var once sync.Once diff --git a/services/core/tests/integration/sandbox_deployment_switch_test.go b/services/core/tests/integration/sandbox_deployment_switch_test.go index 7c88d88c0..37b73464b 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_test.go @@ -190,7 +190,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if err := deploymentExecution(t, w).StartReset(SandboxResetTestContext(t.Context()), id, deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - // AdmissionPaused rejects a valid enrollment without consuming it. Authentication + // A reset rejects a valid enrollment without consuming it. Authentication // still precedes deployment details for invalid or retired credentials. spareNode := node spareNode.NodeID = uuid.NewString() diff --git a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go index e43dd7ae3..af2c98e9f 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go @@ -35,13 +35,13 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil + return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { preparations.Add(1) if fail.Load() { return execution.PreparedRuntimeDeployment{}, errors.New("fixture provider unavailable") } - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: configuration}) ctx, cancel := context.WithCancel(t.Context()) diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index 95538e64a..dcae13238 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -27,10 +27,10 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) start := func() (*execution.Worker, func()) { t.Helper() diff --git a/services/core/tests/integration/sandbox_node_auth_order_http_test.go b/services/core/tests/integration/sandbox_node_auth_order_http_test.go index 67f9d9615..4e152b8a3 100644 --- a/services/core/tests/integration/sandbox_node_auth_order_http_test.go +++ b/services/core/tests/integration/sandbox_node_auth_order_http_test.go @@ -75,7 +75,7 @@ func TestSandboxNodeRoutesAuthenticateBeforeDeploymentState(t *testing.T) { // Once Web claims an installation, still before initialization, another // installation's token gets the same 401 it gets after initialization. - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET installation_id=$1, web_managed=true WHERE singleton=true", claimed); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET installation_id=$1 WHERE singleton=true", claimed); err != nil { t.Fatal(err) } run([]check{ diff --git a/services/core/tests/integration/sandbox_reset_test.go b/services/core/tests/integration/sandbox_reset_test.go index 43316c0e0..cc37abdc4 100644 --- a/services/core/tests/integration/sandbox_reset_test.go +++ b/services/core/tests/integration/sandbox_reset_test.go @@ -251,20 +251,17 @@ func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { } func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { - s, w, process := managerFixture(t, 10, 10) - // Reuse the real placement fixture, then adopt its selection as Web-managed. - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET web_managed=true,local_node_id=NULL`) - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET deployment_generation=1,specification_digest=$1`, SandboxDeploymentTestSpec("docker").Digest("docker")) + s, w, d := managerFixture(t, 10, 10) _, pending := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) tenant, suspended := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - allocation := archiveAllocation(t, w, tenant, suspended, process.InstallationID) + allocation := archiveAllocation(t, w, tenant, suspended, d.InstallationID) runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, allocation.ID) tenant, deleted := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - archiveAllocation(t, w, tenant, deleted, process.InstallationID) + archiveAllocation(t, w, tenant, deleted, d.InstallationID) if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: deleted.ID}); err != nil { t.Fatal(err) } - reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, process.InstallationID, deployment.ResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, d.InstallationID, deployment.ResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -274,7 +271,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { } for _, state := range []string{"preparing", "stale", "epoch", "disconnected"} { runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connected_epoch=(SELECT owner_epoch FROM runtime_deployment)`) - onlineManagerNode(t, s, process.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) switch state { case "preparing": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false`) @@ -298,14 +295,14 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { if view.Reset.Remaining.OnOfflineNodes != want { t.Fatalf("%s presence: %+v", state, view.Reset.Remaining) } - if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != process.LocalNodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { + if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != d.NodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { t.Fatal("offline ownership projection", view.Reset.Remaining) } } - if _, err := deploymentExecution(t, w).CompleteReset(t.Context(), process.InstallationID, 1, reset.Reset.RequestedAt); err == nil { + if _, err := deploymentExecution(t, w).CompleteReset(t.Context(), d.InstallationID, 1, reset.Reset.RequestedAt); err == nil { t.Fatal("offline resources were treated as cleaned") } - if err := deploymentService(t, s).RemoveNode(t.Context(), process.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed node with reset resources", err) } if row, err := sessionAdapter(s).GetEnvironment(t.Context(), pending.TenantID, pending.Environment.ID); err == nil && row.Status == "expired" { diff --git a/services/core/tests/integration/sandbox_specification_lifecycle_test.go b/services/core/tests/integration/sandbox_specification_lifecycle_test.go index 19e9e2513..314e90758 100644 --- a/services/core/tests/integration/sandbox_specification_lifecycle_test.go +++ b/services/core/tests/integration/sandbox_specification_lifecycle_test.go @@ -43,9 +43,15 @@ func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, dep } func specificationNode(t *testing.T, s *Store, view deployment.View) deployment.Enrollment { + t.Helper() + return enrollNode(t, s, view, deployment.Capacity{MaxActive: 4, MaxRetained: 16}) +} + +// enrollNode enrolls an online node with capacity on the committed setup view. +func enrollNode(t *testing.T, s *Store, view deployment.View, capacity deployment.Capacity) deployment.Enrollment { t.Helper() nodes := deploymentService(t, s) - token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 16})) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), capacity)) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/session_deletion_test.go b/services/core/tests/integration/session_deletion_test.go index b62dc8629..724ad4555 100644 --- a/services/core/tests/integration/session_deletion_test.go +++ b/services/core/tests/integration/session_deletion_test.go @@ -396,7 +396,7 @@ func TestSessionDeletionKeepsProvisioningInputPlacementUntilSettled(t *testing.T var current state if err := s.pool.QueryRow(ctx, `SELECT s.deleted_at, p.released_at FROM sessions s JOIN environments e ON e.session_id=s.id JOIN runtime_placements p ON p.environment_id=e.id - WHERE s.id=$1 AND p.node_id=$2`, session.ID, d.LocalNodeID).Scan(¤t.deleted, ¤t.released); err != nil { + WHERE s.id=$1 AND p.node_id=$2`, session.ID, d.NodeID).Scan(¤t.deleted, ¤t.released); err != nil { t.Fatal("missing placement", err) } nodes, err := deploymentService(t, s).ListNodes(ctx) diff --git a/services/core/tests/integration/web_deployment_only_migration_test.go b/services/core/tests/integration/web_deployment_only_migration_test.go new file mode 100644 index 000000000..1bbec6ad9 --- /dev/null +++ b/services/core/tests/integration/web_deployment_only_migration_test.go @@ -0,0 +1,39 @@ +package integration + +import ( + "strings" + "testing" + + "github.com/google/uuid" +) + +// A process-configured deployment refuses the upgrade; a Web-managed one +// keeps its installation and reset in both directions. +func TestWebDeploymentOnlyMigrationRefusesProcessDeployment(t *testing.T) { + db, provider := runtimeNamesMigrationSchema(t) + ctx := t.Context() + if _, err := provider.UpTo(ctx, 92); err != nil { + t.Fatal(err) + } + installation := uuid.NewString() + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2`, installation, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 93); err == nil || !strings.Contains(err.Error(), "configured outside Web setup") { + t.Fatal("process deployment upgraded", err) + } + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET web_managed=true, provider_kind='docker', mode='nodes', generation=1, + admission_paused=true, reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`); err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 93); err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(ctx, 92); err != nil { + t.Fatal(err) + } + var restored bool + if err := db.QueryRowContext(ctx, `SELECT web_managed AND local_node_id IS NULL AND admission_paused AND installation_id=$1 FROM runtime_deployment`, installation).Scan(&restored); err != nil || !restored { + t.Fatal("downgrade lost the Web installation", err) + } +} diff --git a/services/core/tests/integration/worker_fixture_test.go b/services/core/tests/integration/worker_fixture_test.go index 872c77345..790749499 100644 --- a/services/core/tests/integration/worker_fixture_test.go +++ b/services/core/tests/integration/worker_fixture_test.go @@ -4,11 +4,17 @@ import ( "context" "errors" "testing" + "time" + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -110,3 +116,76 @@ func fixtureOwner(s *Store, lease *pgunit.Lease) (execution.Owner, error) { Sessions: sessionExecution, }, nil } + +// webDeployment claims s's deployment for a new installation and selects +// provider on it as Web setup does, then closes its execution lease so a +// Worker can start. It returns the installation. +func webDeployment(t *testing.T, s *Store, provider string) string { + t.Helper() + lease, err := pgunit.AcquireLease(t.Context(), s.pool) + if err != nil { + t.Fatal(err) + } + defer lease.Close(context.Background()) + owner, err := fixtureOwner(s, lease) + if err != nil { + t.Fatal(err) + } + installation := uuid.NewString() + input := sandbox.Selection{Provider: provider, DeploymentSpec: SandboxDeploymentTestSpec(provider)} + if provider == "e2b" { + input = e2bSelection() + } + if err := owner.Deployment.Claim(t.Context(), installation); err != nil { + t.Fatal(err) + } + if _, err := owner.Deployment.Initialize(t.Context(), installation, input); err != nil { + t.Fatal(err) + } + return installation +} + +// webRuntimes is the Worker's sandbox runtimes as cmd/server builds them for +// installation: a deferred provider that runs s's committed Web setup on p. +func webRuntimes(t testing.TB, s *Store, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.RuntimeProvider { + deployments := deploymentService(t, s) + return execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { + setup, err := deployments.Setup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, + CoreURL: "http://core.invalid/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p, Suspension: suspension}, nil + }, unusedPreparation(t)) +} + +// unusedPreparation is the preparer of a test that submits no sandbox +// selection through the Worker; preparing one fails the test. +func unusedPreparation(t testing.TB) execution.RuntimeDeploymentPreparer { + return func(context.Context, deployment.Setup) (execution.PreparedRuntimeDeployment, error) { + t.Error("the test prepared a sandbox selection it did not submit") + return execution.PreparedRuntimeDeployment{}, errors.New("unexpected sandbox selection preparation") + } +} + +// startWebWorker starts the Worker on webRuntimes. +func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.Worker { + t.Helper() + w, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) + if err != nil { + t.Fatal(err) + } + return w +} + +// startNextWorker is startWorkerErr after another owner closed its lease. A +// closed lease stays held until PostgreSQL ends its backend, so startup +// retries ErrLeaseHeld briefly. +func startNextWorker(ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { + for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(20 * time.Millisecond) { + w, err := startWorkerErr(ctx, s, dispatcher) + if !errors.Is(err, pgunit.ErrLeaseHeld) || time.Now().After(deadline) { + return w, err + } + } +} diff --git a/services/core/tests/official_mcp.py b/services/core/tests/official_mcp.py index 716073371..b41d95ffb 100644 --- a/services/core/tests/official_mcp.py +++ b/services/core/tests/official_mcp.py @@ -74,19 +74,22 @@ def verify_mcp_configuration(client, other, expect_error): {"authorization": "synthetic-private"}, {"server_url": "https://mcp.example.invalid/mcp?token=synthetic-private"}): invalid.append({**tool, "transport": {**transport, **changes}}) - # Transports other than HTTP stay unsupported with or without an origin. - stdio = {"type": "stdio", "command": "synthetic-private"} - invalid += [{**minimal, "transport": stdio}, {**tool, "transport": stdio}] + # A pinned stdio transport stays unsupported with or without an origin; a + # malformed one gets the official field error first. + stdio, malformed = {"type": "stdio", "command": "synthetic-private", "cwd": "/"}, {"type": "stdio", "command": "synthetic-private"} + invalid += [{**minimal, "transport": stdio}, {**tool, "transport": stdio}, {**tool, "transport": malformed}] for declaration in invalid: - for operation in ( - lambda: agents.create(model="requested-model", tools=[declaration]), - lambda: sessions.create(agent={"model": "requested-model", "tools": [declaration]}, - input="Verify mcp fixture admission.", environment={"type": "none"}), + for prefix, operation in ( + ("", lambda: agents.create(model="requested-model", tools=[declaration])), + ("agent.", lambda: sessions.create(agent={"model": "requested-model", "tools": [declaration]}, + input="Verify mcp fixture admission.", environment={"type": "none"})), ): error = expect_error(BadRequestError, operation) assert "synthetic-private" not in str(error.body) if declaration["transport"] is stdio: assert error.body["message"] == "MCP currently supports HTTP transport only." + if declaration["transport"] is malformed: + assert error.body["message"] == f"Missing required parameter: '{prefix}tools[0].transport.cwd'." assert {item.id for item in sessions.list()} == before assert {item.id for item in agents.list()} == saved_before print("HTTP MCP: pinned saved/Session projections, omitted/null origins, null/empty allowlists, immutable snapshots and rejected writes passed; no native execution claimed.") diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 34bb368cb..1e7588c76 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -19,14 +19,14 @@ The account key is stored encrypted in Core's database and is write-only. It rea | `command` | `RunCommand` | Runs one bounded command as the Runtime user on a running sandbox whose bootstrap completed; output is limited to 1 MiB per stream | | `validate_deployment` | Deployment setup | Reads the template's builds and requires the exact build to be ready with the configured CPU and memory. Without configured resources the selection adopts the build's CPU and memory. Returns the build's status, CPU, memory and reported disk for Core to record; bounded to 30 seconds | | `list_templates`, `list_builds` | [Configuration discovery](../../../../contracts/agents-api/sandbox-deployment.md#configuration-discovery) | Pages the key's visible templates (`GET /v2/templates`) or one template's ready builds, with a transient key. Results are capped at 200 and write no receipt | -| `observe` | Runtime observations | Up to 100 allocations; see [Observations](#observations) | +| `observe` | Runtime observations | One allocation; see [Observations](#observations) | | `verify_credential` | E2B key replacement | Up to 32 allocation references; see [Credential verification](#credential-verification) | Compatible endpoints must return the SDK 2.51.0 template-list and template-build response models; the helper does not adapt other catalog shapes. E2B has no independently configurable disk limit, and sandbox inspection does not expose a build ID: build provenance comes from the validated create selector. ## Private JSON boundary -[`helper_contract.go`](../../internal/sandbox/e2b/helper_contract.go) owns the adapter-private wire types, version, operation and error vocabulary, and bounds. Its generator projects Python declarations into the helper and template sources, deriving managed-bootstrap fields from the Sandbox Provider types, network access values from `agentnetwork.Policy.Validate`, and the SDK version from the hashed dependency lock. The command-input and observation limits come from their shared Go contracts. The generated modules have no SDK or repository dependency and ship with the frozen helper and protected template startup scripts. +[`helper_contract.go`](../../internal/sandbox/e2b/helper_contract.go) owns the adapter-private wire types, version, operation and error vocabulary, and bounds. Its generator projects Python declarations into the helper and template sources, deriving managed-bootstrap fields from the Sandbox Provider types, network access values from `agentnetwork.Policy.Validate`, and the SDK version from the hashed dependency lock. The command-input limit comes from its shared Go contract. The generated modules have no SDK or repository dependency and ship with the frozen helper and protected template startup scripts. Run `go generate ./services/core/internal/sandbox/e2b` from the repository root after changing these declarations. `make check-e2b-provider` and the Go adapter tests reject stale projections; both languages consume generated valid and invalid exchanges covering wire types, extra fields, operation/reference bounds and managed-bootstrap fields. The helper build copies those fixtures with its source before running the pinned-SDK suite. @@ -56,7 +56,7 @@ Inspection uses SDK metadata and ID reads only. SDK `connect` is never used beca ## Observations -`observe` reads each allocation's sandbox ID from its receipt without taking the allocation lock. It then runs one `GET /sandboxes/metrics` request and one labelled listing of the installation's running sandboxes concurrently, within the caller's deadline; the listing stops once every requested sandbox has appeared. Only a sandbox that the listing confirms for exactly that allocation is reported, with the listing's start time. A malformed metrics point makes only its row unavailable. Observation never connects to, renews or changes a sandbox and writes no receipt. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the field mapping. +`observe` reads the allocation's sandbox ID from its receipt without taking the allocation lock. It then runs one `GET /sandboxes/metrics` request for that sandbox and one listing of running sandboxes with the allocation's labels concurrently, within the caller's deadline; the listing stops once the sandbox has appeared. Only a sandbox that the listing confirms for exactly that allocation is reported, with the listing's start time. A malformed metrics point is unavailable. Observation never connects to, renews or changes a sandbox and writes no receipt. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the field mapping. ## Credential verification diff --git a/services/core/tools/e2b-provider/deployment_test.py b/services/core/tools/e2b-provider/deployment_test.py index 4008a4616..27601bbf8 100644 --- a/services/core/tools/e2b-provider/deployment_test.py +++ b/services/core/tools/e2b-provider/deployment_test.py @@ -66,7 +66,7 @@ def test_custom_endpoint_reaches_metrics_client(self, metrics, client): self.config.update(APIURL='https://sandbox-test.sandbase.ai', Domain='sandbox-test.sandbase.ai') metrics.return_value = SimpleNamespace(status_code=503, parsed=None) with self.assertRaises(Failure): - read_metrics(self.config, ['owned-id'], lambda: 5) + read_metrics(self.config, 'owned-id', lambda: 5) configuration = client.call_args.args[0] self.assertEqual((configuration.api_url, configuration.domain), (self.config['APIURL'], self.config['Domain'])) diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index ebd7fadf2..04d6c13b3 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -5,7 +5,6 @@ MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 -MAX_OBSERVATION_REFERENCES = 100 MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 @@ -14,5 +13,5 @@ PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observations"] +RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] SDK_VERSION = "2.51.0" diff --git a/services/core/tools/e2b-provider/observation_test.py b/services/core/tools/e2b-provider/observation_test.py index eb9c09a89..ef204d6d7 100644 --- a/services/core/tools/e2b-provider/observation_test.py +++ b/services/core/tools/e2b-provider/observation_test.py @@ -1,4 +1,4 @@ -"""Read-only batch observation; live metrics qualification remains separate.""" +"""Read-only observation; live metrics qualification remains separate.""" from datetime import datetime, timezone from types import SimpleNamespace from unittest.mock import patch @@ -10,9 +10,8 @@ class ObservationTest(ProviderTest): - def observe(self, references): - request = dict(self.request, Operation='observe', References=references, - Reference={key: '' for key in self.reference}) + def observe(self, reference=None): + request = dict(self.request, Operation='observe', Reference=reference or self.reference) try: return Provider(request).execute() except Failure as error: @@ -30,51 +29,47 @@ def next_items(**options): self.api.list.return_value = paginator return paginator - def test_one_metrics_request_maps_owned_running_sandboxes(self): + def test_one_metrics_request_maps_the_owned_running_sandbox(self): self.assertEqual(self.call('create')['ErrorCode'], '') self.cloud.started_at = datetime(2026, 9, 25, 10, 31, 34, tzinfo=timezone.utc) other = SimpleNamespace(sandbox_id='other-id', metadata={'oac_installationid': self.config['InstallationID']}) paginator = self.listing([other, self.cloud], [other]) - stopped = {key: str(uuid4()) for key in self.reference} point = {'cpuCount': 2, 'cpuUsedPct': 19.55, 'memUsed': 183836672, 'memTotal': 2079141888, 'memCache': 1, 'diskUsed': 1593188352, 'diskTotal': 23511863296, 'timestamp': '2026-09-25T10:31:36.667115804Z', 'timestampUnix': 1790332296} - with patch('provider.read_metrics', return_value={'owned-id': point}) as metrics, \ + with patch('provider.read_metrics', return_value=point) as metrics, \ patch('provider.Receipt') as receipt: - result = self.observe([self.reference, stopped]) + result = self.observe() self.assertEqual(result['ErrorCode'], '') metrics.assert_called_once() - self.assertEqual(metrics.call_args.args[1], ['owned-id']) + self.assertEqual(metrics.call_args.args[1], 'owned-id') receipt.assert_not_called() self.api.connect.assert_not_called() self.api.set_timeout.assert_not_called() - self.assertEqual(self.api.list.call_args.kwargs['query'].metadata, - {'oac_installationid': self.config['InstallationID']}) - owned, missing = result['Observations'] - self.assertEqual(owned, dict(self.reference, Status='observed', + self.assertEqual(self.api.list.call_args.kwargs['query'].metadata, self.cloud.metadata) + self.assertEqual(result['Observation'], dict(Status='observed', ObservedAt='2026-09-25T10:31:36.667115+00:00', StartedAt='2026-09-25T10:31:34+00:00', CPUCount=2, CPUUsedPct=19.55, MemUsed=183836672, MemTotal=2079141888, DiskUsed=1593188352, DiskTotal=23511863296)) - self.assertEqual(missing, dict(stopped, Status='unavailable')) - self.assertEqual(paginator.reads, 1, 'listing continued after every receipt sandbox was found') + self.assertEqual(paginator.reads, 1, 'listing continued after the receipt sandbox was found') + stopped = {key: str(uuid4()) for key in self.reference} + self.assertEqual(self.observe(stopped)['Observation'], {'Status': 'unavailable'}) del point['diskUsed'] point['memTotal'] = -1 - with patch('provider.read_metrics', return_value={'owned-id': point}): + with patch('provider.read_metrics', return_value=point): self.listing([self.cloud]) - self.assertEqual(self.observe([self.reference])['Observations'], [dict(self.reference, Status='unavailable')]) + self.assertEqual(self.observe()['Observation'], {'Status': 'unavailable'}) point['memTotal'] = 2079141888 - with patch('provider.read_metrics', return_value={'owned-id': point}): + with patch('provider.read_metrics', return_value=point): self.listing([self.cloud]) - row = self.observe([self.reference])['Observations'][0] + row = self.observe()['Observation'] self.assertEqual((row['Status'], row['DiskUsed'], row['DiskTotal']), ('observed', None, None)) - def test_absent_listing_is_not_running_and_rejects_oversized_batches(self): + def test_absent_listing_is_not_running(self): self.assertEqual(self.call('create')['ErrorCode'], '') self.listing([]) - with patch('provider.read_metrics', return_value={}): - result = self.observe([self.reference]) - self.assertEqual(result['Observations'], [dict(self.reference, Status='not_running')]) - references = [{key: str(uuid4()) for key in self.reference} for _ in range(101)] - self.assertEqual(self.observe(references)['ErrorCode'], 'invalid') + with patch('provider.read_metrics', return_value=None): + result = self.observe() + self.assertEqual(result['Observation'], {'Status': 'not_running'}) diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index dbf3111f0..9b2fcafe7 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -1,5 +1,5 @@ """Five bounded SDK operations for an already authorized Core allocation, plus -read-only deployment validation and batch observation.""" +read-only deployment validation and observation.""" from concurrent.futures import ThreadPoolExecutor import json import math @@ -14,7 +14,7 @@ from sdk import connection_material, definitely_rejected, list_builds, list_templates, read_metrics, restore, run, sdk_options, validate_deployment, verify_team_template from state import Failure, Receipt, private_root, read_receipt from helper_contract_generated import (PROTOCOL_VERSION, OPERATIONS, REQUEST_FIELDS, REFERENCE_FIELDS, - MAX_OBSERVATION_REFERENCES, MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS) + MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS) PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) @@ -38,9 +38,9 @@ def utc(value): return value.astimezone(timezone.utc).isoformat() -def observed(reference, cloud, point): - """Map one metrics point without changing E2B units. A malformed point makes - only its own row unavailable. +def observed(cloud, point): + """Map one metrics point without changing E2B units. A malformed point is + unavailable. Disk metrics need a newer envd; unless E2B reports both integer values and a positive total, disk stays unknown rather than an observed zero.""" @@ -56,11 +56,11 @@ def observed(reference, cloud, point): not math.isfinite(cpu_pct) or cpu_pct < 0 or any(type(v) is not int or v < 0 for v in values) or metric.mem_total < 1): raise ValueError('malformed metrics point') - return dict(reference, Status='observed', ObservedAt=utc(metric.timestamp), StartedAt=utc(cloud.started_at), + return dict(Status='observed', ObservedAt=utc(metric.timestamp), StartedAt=utc(cloud.started_at), CPUCount=cpu_count, CPUUsedPct=cpu_pct, MemUsed=metric.mem_used, MemTotal=metric.mem_total, DiskUsed=metric.disk_used if disk_known else None, DiskTotal=metric.disk_total if disk_known else None) except Exception: - return dict(reference, Status='unavailable') + return {'Status': 'unavailable'} class Provider: @@ -80,12 +80,8 @@ def __init__(self, request): if (type(request['Version']) is not int or request['Version'] != PROTOCOL_VERSION or not isinstance(request['Operation'], str) or request['Operation'] not in OPERATIONS or set(request) - set(REQUEST_FIELDS) or - (request['Operation'] not in ('validate_deployment', 'observe', 'list_templates', 'list_builds', 'verify_credential') and + (request['Operation'] not in ('validate_deployment', 'list_templates', 'list_builds', 'verify_credential') and not valid_reference(self.reference)) or - (request['Operation'] == 'observe' and - (not 1 <= len(self.references) <= MAX_OBSERVATION_REFERENCES or - not all(valid_reference(r) for r in self.references) or - len({tuple(sorted(r.items())) for r in self.references}) != len(self.references))) or (request['Operation'] == 'verify_credential' and (len(self.references) > MAX_CREDENTIAL_REFERENCES or not all(valid_reference(r) for r in self.references))) or @@ -279,66 +275,38 @@ def kill(self): self.receipt.save(status='killed', settled=True, bootstrap_complete=False, connection=None) def observe(self): - """Latest metrics of owned running sandboxes, without locks, writes or connect. + """Latest metrics of the owned running sandbox, without locks, writes or connect. - Receipts name each allocation's sandbox so that the one batch metrics - request runs alongside the labelled listing that confirms it is running.""" - root = private_root(self.config) - statuses, candidates = [], {} - for index, reference in enumerate(self.references): - try: - record = read_receipt(self.config, root, reference) or {} - except Failure as error: - statuses.append(error.code) - continue - except Exception: - statuses.append('unavailable') - continue - ids = record.get('ids', []) - if record.get('status') in ('killed', 'rejected'): - statuses.append('not_running') - elif len(ids) == 1 and isinstance(ids[0], str): - statuses.append('unavailable') - candidates[index] = ids[0] - else: - statuses.append('unavailable') - if not candidates: - return [dict(r, Status=status) for r, status in zip(self.references, statuses)] - installation = self.config['InstallationID'] - # One allocation filters by all its labels; a page lists the installation. - metadata = {PREFIX + 'installationid': installation} - if len(self.references) == 1: - metadata = self.metadata_for(self.references[0]) - wanted, seen = set(candidates.values()), set() + The receipt names the allocation's sandbox so that the metrics request + runs alongside the labelled listing that confirms it is running.""" + try: + record = read_receipt(self.config, private_root(self.config), self.reference) or {} + except Failure as error: + return {'Status': error.code} + except Exception: + return {'Status': 'unavailable'} + ids = record.get('ids', []) + if record.get('status') in ('killed', 'rejected'): + return {'Status': 'not_running'} + if len(ids) != 1 or not isinstance(ids[0], str): + return {'Status': 'unavailable'} + found = [] with ThreadPoolExecutor(max_workers=1) as pool: - metrics = pool.submit(read_metrics, self.config, sorted(wanted), self.remaining) - running = {} - paginator = Sandbox.list(query=SandboxQuery(metadata=metadata, state=[SandboxState.RUNNING]), + metrics = pool.submit(read_metrics, self.config, ids[0], self.remaining) + paginator = Sandbox.list(query=SandboxQuery(metadata=self.metadata, state=[SandboxState.RUNNING]), limit=100, **self.options()) - # Stop once every receipt's sandbox has been listed. Detection of a + # Stop once the receipt's sandbox has been listed. Detection of a # second sandbox with the same allocation labels then covers only # the pages read; lifecycle discovery remains exhaustive. - while paginator.has_next and not wanted <= seen: - for cloud in paginator.next_items(**self.options()): - labels = cloud.metadata or {} - if labels.get(PREFIX + 'installationid') == installation: - key = tuple(labels.get(PREFIX + field.lower()) for field in FIELDS[1:]) - running.setdefault(key, []).append(cloud) - seen.add(cloud.sandbox_id) - points = metrics.result() - result = [] - for index, reference in enumerate(self.references): - if index not in candidates: - result.append(dict(reference, Status=statuses[index])) - continue - found = running.get(tuple(reference[field] for field in FIELDS[1:]), []) - if not found: - result.append(dict(reference, Status='not_running')) - elif len(found) != 1 or found[0].sandbox_id != candidates[index] or not isinstance(points.get(candidates[index]), dict): - result.append(dict(reference, Status='unavailable')) - else: - result.append(observed(reference, found[0], points[candidates[index]])) - return result + while paginator.has_next and not any(cloud.sandbox_id == ids[0] for cloud in found): + found += [cloud for cloud in paginator.next_items(**self.options()) + if all((cloud.metadata or {}).get(k) == v for k, v in self.metadata.items())] + point = metrics.result() + if not found: + return {'Status': 'not_running'} + if len(found) != 1 or found[0].sandbox_id != ids[0] or not isinstance(point, dict): + return {'Status': 'unavailable'} + return observed(found[0], point) def metadata_for(self, reference): return {PREFIX + field.lower(): value for field, value in @@ -397,7 +365,7 @@ def execute(self): self.verify_credential() return {'Version': PROTOCOL_VERSION, 'DeploymentValid': True, 'ErrorCode': ''} if self.q['Operation'] == 'observe': - return {'Version': PROTOCOL_VERSION, 'Observations': self.observe(), 'ErrorCode': ''} + return {'Version': PROTOCOL_VERSION, 'Observation': self.observe(), 'ErrorCode': ''} verify_team_template(self.config, self.remaining) build = validate_deployment(self.config, self.remaining) return {'Version': PROTOCOL_VERSION, 'DeploymentValid': True, 'TemplateBuild': build, 'ErrorCode': ''} diff --git a/services/core/tools/e2b-provider/sdk.py b/services/core/tools/e2b-provider/sdk.py index c58236a2a..4b2d3afb7 100644 --- a/services/core/tools/e2b-provider/sdk.py +++ b/services/core/tools/e2b-provider/sdk.py @@ -18,7 +18,7 @@ from state import Failure -from helper_contract_generated import SDK_VERSION, MAX_OUTPUT, MAX_COMMAND_INPUT, MAX_OBSERVATION_REFERENCES +from helper_contract_generated import SDK_VERSION, MAX_OUTPUT, MAX_COMMAND_INPUT def list_templates(config, remaining): @@ -120,16 +120,14 @@ def validate_deployment(config, remaining): raise Failure('unconfirmed') -def read_metrics(config, sandbox_ids, remaining): - """Latest metrics point per sandbox from one batch request of at most 100 IDs.""" - if not 1 <= len(sandbox_ids) <= MAX_OBSERVATION_REFERENCES: - raise Failure('invalid') +def read_metrics(config, sandbox_id, remaining): + """Latest metrics point of one sandbox, or None when E2B reports none.""" client = get_api_client(ConnectionConfig(**sdk_options(config, remaining))) - response = get_sandboxes_metrics.sync_detailed(client=client, sandbox_ids=sandbox_ids) + response = get_sandboxes_metrics.sync_detailed(client=client, sandbox_ids=[sandbox_id]) if (response.status_code != 200 or not isinstance(response.parsed, SandboxesWithMetrics) or not isinstance(response.parsed.sandboxes, dict)): raise Failure('unconfirmed') - return response.parsed.sandboxes + return response.parsed.sandboxes.get(sandbox_id) def connection_material(sandbox): diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index aa79da367..0e2c8e8c6 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -22,19 +22,12 @@ {"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, {"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"duplicate-observation","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, {"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, {"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe-count-0","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"observe-count-100","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe-count-101","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000022-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000023-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000024-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000025-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000026-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000027-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000028-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000029-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000002f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000030-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000031-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000032-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000033-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000034-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000035-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000036-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000037-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000038-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000039-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000003f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000040-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000041-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000042-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000043-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000044-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000045-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000046-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000047-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000048-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000049-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000004f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000050-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000051-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000052-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000053-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000054-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000055-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000056-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000057-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000058-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000059-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000005f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000060-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000061-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000062-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000063-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000064-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000065-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, -{"kind":"request","name":"observe-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"observe","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, +{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","NetworkAccess":"enabled","AllowedDomains":null},"RuntimeBootstrap":{"version":1,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, {"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, {"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","AllowedDomains":null,"CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","NetworkAccess":"enabled","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","version":1},"Version":1},"valid":false}, diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index 554b4fad8..7235c8274 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -1,6 +1,6 @@ # microsandbox Sandbox Provider helper -microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the provider-neutral `sandbox.CheckpointProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. +microsandbox runs each hosted Session in its own microVM on a Linux amd64 node with KVM, and it is the Sandbox Provider that supports idle suspension. Core forwards provider operations to the node over the [node protocol](../../../../contracts/agents-api/node-generation-protocol.md); the node's adapter ([`sandbox/microsandbox`](../../internal/sandbox/microsandbox)) runs this helper once per operation. The helper links the microsandbox Go SDK v0.7.2 with its FFI library, so Core and the node program stay CGO-free Go binaries. It implements the checkpoint operations of the provider-neutral `sandbox.SandboxProvider` with full snapshots. It has no daemon, lifecycle database, scheduler or network control plane. [Add a Sandbox Provider](../../../../docs/sandbox-provider.md) owns the provider contract. [Sandbox deployment](../../../../contracts/agents-api/sandbox-deployment.md) owns the resources, Runtime release and suspension policy; the [nodes guide](../../../../docs/getting-started/nodes.md) owns node installation, host requirements, the node's directories and its network policy.