From 4f6616f2fdf6d989c3391c5b277bba516e11c2ca Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 7 Oct 2026 17:36:17 +0000 Subject: [PATCH] Delete the process deployment path --- .../web/src/features/sandbox/NodeList.test.ts | 2 - apps/web/src/features/sandbox/NodeList.tsx | 5 +- apps/web/src/lib/locale-strings.ts | 2 - contracts/agents-api/core.openapi.yaml | 2 +- docs/web/console-api-usage.md | 2 +- docs/zh/web/console-api-usage.md | 4 +- services/core/cmd/server/managed_setup.go | 4 +- .../core/cmd/server/managed_setup_test.go | 4 +- .../internal/api/core_error_catalog_test.go | 2 - .../core/internal/api/errors_deployment.go | 2 - services/core/internal/api/errors_test.go | 1 - .../internal/api/placement_errors_test.go | 2 +- .../internal/api/sandbox_deployment_setup.go | 2 +- .../core/internal/api/sandbox_manager_test.go | 9 - .../db/queries/runtime_deployment.sql | 4 - .../db/queries/runtime_lifecycle_nodes.sql | 2 +- .../internal/db/queries/runtime_nodes.sql | 3 - .../db/queries/runtime_suspension.sql | 8 - .../db/queries/sandbox_deployment_setup.sql | 2 +- .../internal/db/queries/sandbox_reset.sql | 6 +- services/core/internal/db/sqlc/models.go | 3 - .../db/sqlc/runtime_deployment.sql.go | 23 +- .../db/sqlc/runtime_lifecycle_nodes.sql.go | 2 +- .../internal/db/sqlc/runtime_nodes.sql.go | 19 +- .../db/sqlc/runtime_suspension.sql.go | 24 -- .../db/sqlc/sandbox_deployment_setup.sql.go | 2 +- .../internal/db/sqlc/sandbox_reset.sql.go | 13 +- .../internal/deployment/allocations_test.go | 10 +- services/core/internal/deployment/errors.go | 1 - .../core/internal/deployment/execution.go | 121 +------- .../core/internal/deployment/fakes_test.go | 98 ++----- services/core/internal/deployment/nodes.go | 10 +- .../deployment/placement/placement.go | 30 +- .../deployment/placement/placement_test.go | 12 +- services/core/internal/deployment/rules.go | 2 +- .../core/internal/deployment/rules_test.go | 8 +- services/core/internal/deployment/service.go | 4 +- .../core/internal/deployment/service_test.go | 16 +- .../internal/deployment/session_archive.go | 4 +- .../deployment/session_archive_test.go | 9 +- services/core/internal/deployment/setup.go | 16 +- services/core/internal/deployment/storage.go | 18 +- .../execution/deployment_fixture_test.go | 67 ++--- .../core/internal/execution/owner_test.go | 4 +- .../internal/execution/runtime_compute.go | 34 +-- .../internal/execution/runtime_lifecycle.go | 79 ++--- .../internal/execution/runtime_manager.go | 14 +- .../execution/runtime_manager_test.go | 3 +- .../internal/execution/runtime_pending.go | 3 - .../runtime_retirement_failure_test.go | 8 +- .../sandbox_deployment_drain_test.go | 4 +- .../execution/sandbox_deployment_setup.go | 25 +- .../sandbox_deployment_setup_test.go | 14 +- .../execution/sandbox_deployment_switch.go | 4 +- .../sandbox_deployment_switch_test.go | 6 +- .../sandbox_provider_contract_test.go | 2 +- .../core/internal/execution/sandbox_reset.go | 3 - .../internal/execution/sandbox_reset_test.go | 10 +- .../execution/sandbox_snapshot_budget_test.go | 2 +- services/core/internal/execution/worker.go | 10 +- .../postgres/deploymentpg/allocations.go | 16 - .../postgres/deploymentpg/records.go | 4 +- .../postgres/deploymentpg/reset_test.go | 6 +- .../postgres/deploymentpg/setup_test.go | 74 +---- .../deploymentpg/specification_test.go | 9 +- .../persistence/postgres/deploymentpg/tx.go | 19 -- .../postgres/placementpg/placementpg.go | 3 +- .../postgres/sessionpg/creation_test.go | 13 +- .../internal/sandbox/providers/registry.go | 9 - .../sandbox/providers/registry_test.go | 13 +- .../core/internal/sessions/creation_test.go | 2 +- .../migrations/000093_web_deployment_only.sql | 78 +++++ .../admin_session_archive_race_test.go | 8 - .../admin_session_archive_worker_http_test.go | 2 +- .../credential_matrix_http_test.go | 4 +- .../device_bootstrap_binding_test.go | 2 +- ...sted_initialization_failure_public_test.go | 25 +- .../integration/runtime_adoption_test.go | 73 ----- .../runtime_capabilities_pending_test.go | 4 +- .../runtime_compute_lifecycle_test.go | 71 +---- .../runtime_configuration_cleanup_test.go | 2 +- .../integration/runtime_connection_test.go | 7 +- .../runtime_creation_settlement_test.go | 6 +- .../integration/runtime_deployment_test.go | 275 +++--------------- .../runtime_deployment_worker_test.go | 32 +- .../runtime_initialization_test.go | 11 +- .../runtime_input_admission_test.go | 15 +- .../runtime_lifecycle_nodes_test.go | 25 +- .../integration/runtime_lifecycle_test.go | 42 +-- .../runtime_node_lifecycle_fixture_test.go | 15 +- .../tests/integration/runtime_nodes_test.go | 71 ++--- .../integration/runtime_observation_test.go | 2 +- .../tests/integration/runtime_pending_test.go | 7 +- .../tests/integration/runtime_scan_test.go | 6 +- .../integration/runtime_suspension_test.go | 43 +-- .../runtime_wake_hint_integration_test.go | 10 +- .../sandbox_deployment_switch_test.go | 2 +- .../sandbox_deployment_switch_worker_test.go | 4 +- .../sandbox_deployment_worker_test.go | 4 +- .../sandbox_node_auth_order_http_test.go | 2 +- .../tests/integration/sandbox_reset_test.go | 19 +- .../sandbox_specification_lifecycle_test.go | 8 +- .../integration/session_deletion_test.go | 2 +- .../web_deployment_only_migration_test.go | 39 +++ .../tests/integration/worker_fixture_test.go | 79 +++++ 105 files changed, 587 insertions(+), 1381 deletions(-) create mode 100644 services/core/migrations/000093_web_deployment_only.sql delete mode 100644 services/core/tests/integration/runtime_adoption_test.go create mode 100644 services/core/tests/integration/web_deployment_only_migration_test.go diff --git a/apps/web/src/features/sandbox/NodeList.test.ts b/apps/web/src/features/sandbox/NodeList.test.ts index d32fab58e..d90258500 100644 --- a/apps/web/src/features/sandbox/NodeList.test.ts +++ b/apps/web/src/features/sandbox/NodeList.test.ts @@ -21,8 +21,6 @@ describe("node state", () => { expect(nodeState(node("a", { online: false }), [], true, core)).toBe("unconfirmed"); expect(nodeState(node("a", { core_url: "https://core-old.example" }), [], false, core)).toBe("old_address"); expect(nodeState(node("a", { online: false, core_url: "https://core-old.example" }), [], false, core)).toBe("old_address"); - // A node Core did not enroll, such as a file-managed local one, reports no address: unknown, not old. - expect(nodeState(node("a", { core_url: "" }), [], false, core)).toBe("available"); expect(nodeState(node("a", { online: false, cleanup_pending: 2 }), [], false, core)).toBe("offline"); expect(nodeState(node("a", { provider_ready: false }), [], false, core)).toBe("degraded"); }); diff --git a/apps/web/src/features/sandbox/NodeList.tsx b/apps/web/src/features/sandbox/NodeList.tsx index aa8bb7d50..b5c4fd979 100644 --- a/apps/web/src/features/sandbox/NodeList.tsx +++ b/apps/web/src/features/sandbox/NodeList.tsx @@ -15,11 +15,10 @@ export type NodeState = "unconfirmed" | "old_address" | "offline" | "degraded" | /** * Whether a node enrolled with another Core address than the deployment's - * `coreUrl`. An empty address is unknown, not old: a node Core did not enroll, - * such as a file-managed local one, reports none. + * `coreUrl`. While the deployment is unknown, no node is on an old address. */ export function onOldAddress(node: SandboxNode, coreUrl: string): boolean { - return Boolean(node.core_url && coreUrl && node.core_url !== coreUrl); + return coreUrl !== "" && node.core_url !== coreUrl; } /** diff --git a/apps/web/src/lib/locale-strings.ts b/apps/web/src/lib/locale-strings.ts index 880588c9c..e6080a26d 100644 --- a/apps/web/src/lib/locale-strings.ts +++ b/apps/web/src/lib/locale-strings.ts @@ -203,7 +203,6 @@ export const chinese = { "Couldn't confirm the sandbox change": "沙箱更改未能确认", "Sandbox state couldn't be read": "无法读取沙箱状态", "Nodes": "节点", - "Local nodes run on the Core server. Capacity and counts are reported by Core.": "本地节点运行在 Core 服务器上。容量和资源数量由 Core 上报。", "Sandbox nodes": "沙箱节点", "Node": "节点", "Health": "健康状态", @@ -244,7 +243,6 @@ export const chinese = { "Automatic placement": "自动分配", "available": "可用", "unavailable": "不可用", - "Optional. Local nodes run on the Core server. A selected node must be available; Core will not fall back to another node.": "可选。本地节点运行在 Core 服务器上。所选节点必须可用；Core 不会自动改用其他节点。", "Loading nodes…": "正在加载节点…", "Retry directory": "重新加载节点目录", "No nodes are registered.": "尚未注册节点。", diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index c04c1097c..27637917c 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -6222,7 +6222,7 @@ paths: post: consumes: - application/json - description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. + description: Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. parameters: - description: Deployment selection in: body diff --git a/docs/web/console-api-usage.md b/docs/web/console-api-usage.md index b81088cf4..09df51a49 100644 --- a/docs/web/console-api-usage.md +++ b/docs/web/console-api-usage.md @@ -98,7 +98,7 @@ The list carries each harness's configuration, so the console does not read `GET | Deployment | `GET`, `POST`, `PUT /core/v1/sandbox/deployment` | Read the provider, the read-only `core_url` (`OAC_PUBLIC_URL`, shown in the setup review and never sent), reset state, installation ID and specification; a 409 `sandbox_configuration_error` (E2B with a loopback `public_url`) shows the shared client's fixed safe address-configuration message in the setup wizard, with Managed in System leading to System, and leaves nothing to confirm; initialize the deployment with `resources` and the Docker or microsandbox `runtime` release, or with the E2B account and no `resources` (Core adopts the template build's CPU and memory); change its settings with the expected generation. E2B's `metadata.template_build` (status, CPU, memory, disk) shows on System, the Sandbox configuration summary and Sandbox metrics, and sizes each sandbox when `specification.resources` is missing; microsandbox's `suspension` (idle and retention seconds) shows on System and the Nodes summary | | E2B discovery | `POST /core/v1/sandbox/providers/e2b/discovery` | The setup wizard lists the templates the entered E2B key can see, then the selected template's ready builds. The key travels only in these request bodies and the deployment write | | Reset | `POST`, `DELETE /core/v1/sandbox/deployment/reset` | Explicitly clear hosted resources, or cancel the remaining clear at the observed generation; show Core's remaining and offline projection | -| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health; an empty `core_url` (a node Core did not enroll) is unknown, not old. **Add node** follows only the node whose `enrollment_id` equals its command's | +| Nodes | `GET /core/v1/sandbox/nodes` | Nodes page; fleet on Overview; node capacity on Sandbox metrics. An online node's `diagnostic` (`docker_unavailable`, `docker_limits_unsupported`, `runtime_image_unavailable`, `kvm_unavailable`, `microsandbox_artifacts_unavailable`, `capacity_insufficient`, `provider_unavailable`; any other value reads as `provider_unavailable`) marks it degraded and names the reason and fix in the help tip beside its status on each of these and on the node's page. A node whose `core_url` (the address it enrolled with) differs from the deployment's `core_url` is named on the Nodes page as bound to an old address, to be removed and added again, and its status there and on its page reads Old address instead of its health. **Add node** follows only the node whose `enrollment_id` equals its command's | | Node detail | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics node dialog: the host's CPU busy share and memory from its last heartbeat, and their history over the page's range. **Edit node** reads `host.effective_cpu_cores` and `host.total_memory_bytes` to show the host beside each sandbox's size and at most how many of those fit | | Allocations | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes page; Sandbox metrics. Under microsandbox, a node's page shows from `compute_phase_changed_at` how long each allocation has been in its compute phase and, while suspended, about when Core reclaims it (that time plus the deployment's `suspension.retention_seconds`); a null time shows a dash | | Enrollment | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**: the administrator sets the node's sandbox limits (`max_active`; `max_retained` only for microsandbox, equal to `max_active` for Docker) before Core issues a single-use token inside a command that verifies the installer checksum, with the command's `enrollment_id`, which the node it registers reports. The command runs the installer with sudo (a system service) and passes the token on standard input; root runs it directly. No ordinary-user installation or removal entry is exposed, and the log hint always names the system service. The command downloads the installer from the installation's `public_url`. No token is requested until the installation is read, when it cannot be read, when it is `local_only` (or its `public_url` is not an HTTPS origin), or when `/console/config` lists `node_artifacts` without the deployment's provider. The dialog reads both again on opening and when the window regains focus | diff --git a/docs/zh/web/console-api-usage.md b/docs/zh/web/console-api-usage.md index a6037092a..1b4890fd5 100644 --- a/docs/zh/web/console-api-usage.md +++ b/docs/zh/web/console-api-usage.md @@ -1,7 +1,7 @@ --- title: "控制台 API 使用" source: docs/web/console-api-usage.md -source_hash: 50edc63c79976e9aad9590604a0e361aae178144bc8a6009989b2da5d031408d +source_hash: 7318d1d082d19cc1681a2bd91556c4dc7e16b211ae7cd3936b99d63c26949900 --- 本页列出各控制台页面读取和写入的 Core 路由，以及控制台如何限定读取范围。[administrator API contract](../../../contracts/agents-api/zh/admin-api.md) 定义了路由、响应结构、分页和审计记录；[API namespaces and credentials](../api/index.md) 定义了本文使用的术语。 @@ -100,7 +100,7 @@ source_hash: 50edc63c79976e9aad9590604a0e361aae178144bc8a6009989b2da5d031408d | 部署 | `GET`、`POST`、`PUT /core/v1/sandbox/deployment` | 读取提供商、只读 `core_url`（即 `OAC_PUBLIC_URL`，会显示在设置审核中且绝不发送）、重置状态、安装 ID 和规范；409 `sandbox_configuration_error`（E2B 搭配回环地址形式的 `public_url`）会在设置向导中显示共享客户端固定的安全地址配置消息，并通过 Managed in System 前往 System，且无需确认；使用 `resources` 以及 Docker 或 microsandbox 的 `runtime` release 初始化部署，或者使用 E2B 账户且不提供 `resources`（Core 采用模板构建的 CPU 和内存）；使用预期的 generation 更改设置。E2B 的 `metadata.template_build`（状态、CPU、内存、磁盘）会显示在 System、Sandbox 配置摘要和 Sandbox metrics 中；当缺少 `specification.resources` 时，它还会确定每个 Sandbox 的大小；microsandbox 的 `suspension`（空闲和保留秒数）会显示在 System 和 Nodes 摘要中 | | E2B 发现 | `POST /core/v1/sandbox/providers/e2b/discovery` | 设置向导先列出输入的 E2B 密钥可见的模板，再列出所选模板的可用构建。该密钥只会通过这些请求体和部署写入请求传输 | | 重置 | `POST`、`DELETE /core/v1/sandbox/deployment/reset` | 显式清除托管资源，或在观测到的 generation 处取消剩余清除；显示 Core 的剩余资源和离线预测 | -| Nodes | `GET /core/v1/sandbox/nodes` | Nodes 页面；Overview 上的机群；Sandbox metrics 中的节点容量。在线节点的 `diagnostic`（`docker_unavailable`、`docker_limits_unsupported`、`runtime_image_unavailable`、`kvm_unavailable`、`microsandbox_artifacts_unavailable`、`capacity_insufficient`、`provider_unavailable`；任何其他值均读取为 `provider_unavailable`）会将其标记为降级，并在上述每个页面及节点页面中，紧邻状态的帮助提示里说明原因和修复方法。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，Nodes 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 Old address，而不是健康状态；如果 `core_url` 为空（Core 未注册该节点），则状态为未知，而不是旧地址。**Add node** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | +| Nodes | `GET /core/v1/sandbox/nodes` | Nodes 页面；Overview 上的机群；Sandbox metrics 中的节点容量。在线节点的 `diagnostic`（`docker_unavailable`、`docker_limits_unsupported`、`runtime_image_unavailable`、`kvm_unavailable`、`microsandbox_artifacts_unavailable`、`capacity_insufficient`、`provider_unavailable`；任何其他值均读取为 `provider_unavailable`）会将其标记为降级，并在上述每个页面及节点页面中，紧邻状态的帮助提示里说明原因和修复方法。如果节点的 `core_url`（其注册时使用的地址）与部署的 `core_url` 不同，Nodes 页面会将其标记为绑定到旧地址，需要移除后重新添加；此时它在该页面和节点页面中的状态会显示 Old address，而不是健康状态。**Add node** 仅跟踪 `enrollment_id` 与其命令所含 `enrollment_id` 相等的节点 | | 节点详情 | `GET /core/v1/sandbox/nodes/{node_id}?range=1h\|6h\|24h` | Sandbox metrics 节点对话框：主机自最近一次心跳以来的 CPU 忙碌占比和内存使用量，以及页面所选范围内二者的历史记录。**Edit node** 读取 `host.effective_cpu_cores` 和 `host.total_memory_bytes`，用于在每个 Sandbox 大小旁显示主机容量，以及最多可容纳多少个该大小的 Sandbox | | 分配 | `GET /core/v1/sandbox/nodes/{node_id}/allocations` | Nodes 页面；Sandbox metrics。在 microsandbox 下，节点页面根据 `compute_phase_changed_at` 显示每个分配处于计算阶段的时间，并在分配暂停时估算 Core 回收它的时间（该时间加上部署的 `suspension.retention_seconds`）；时间为 null 时显示短横线 | | 注册 | `POST /core/v1/sandbox/enrollment-tokens` | **Add node**：管理员先设置节点的 Sandbox 限制（`max_active`；`max_retained` 仅适用于 microsandbox，在 Docker 下等于 `max_active`），然后 Core 才会把一次性令牌放入命令中；该命令会验证安装程序校验和，并包含命令的 `enrollment_id`，节点注册时会报告此 ID。命令使用 sudo 运行安装程序（作为系统服务），并通过标准输入传递令牌；以 root 运行时则直接执行。界面不提供普通用户安装或移除入口，日志提示始终指明系统服务。命令从安装的 `public_url` 下载安装程序。只有成功读取安装信息后才会请求令牌；如果安装信息无法读取、安装为 `local_only`（或其 `public_url` 不是 HTTPS 来源），或者 `/console/config` 列出的 `node_artifacts` 不包含部署的提供商，则不会请求令牌。对话框在打开时和窗口重新获得焦点时，会再次读取这两项信息 | diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index 0ca908728..8606525d7 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -145,11 +145,11 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared if err != nil { return execution.PreparedRuntimeDeployment{}, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) } - selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, + selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: s.runtimeAPI, BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, - Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} + Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second} } return execution.PreparedRuntimeDeployment{Config: selected, Publish: s.publish}, nil } diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index d6d2ff663..54e21dc1a 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -177,9 +177,9 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { t.Fatal("preparation published or lost candidate configuration") } committed := *candidate.Config - committed.Generation, committed.AdmissionPaused = 2, true + committed.Generation = 2 candidate.Publish(&committed) - if got := s.selected.Load(); got.Generation != 2 || got.Config.ProviderKind != "microsandbox" || !got.Config.AdmissionPaused { + if got := s.selected.Load(); got.Generation != 2 || got.Config.ProviderKind != "microsandbox" { t.Fatal("commit did not publish the validated selection") } } diff --git a/services/core/internal/api/core_error_catalog_test.go b/services/core/internal/api/core_error_catalog_test.go index 2d81518ab..42d36dee5 100644 --- a/services/core/internal/api/core_error_catalog_test.go +++ b/services/core/internal/api/core_error_catalog_test.go @@ -24,8 +24,6 @@ var nonAdministrationCodes = []string{ "environment_input_cancelled", "environment_input_expired", "model_provider_required", "sandbox_nodes_preparing", "turn_conflict", // Machine routes for nodes and native installers. "installation_authorization_invalid", "installation_unavailable", "invalid_node_credential", "sandbox_node_address_mismatch", - // Removal of the file-managed local node, which the process deployment path owns. - "runtime_local_node_configured", } // The shared catalog lists every code an administration caller (/core/v1 or diff --git a/services/core/internal/api/errors_deployment.go b/services/core/internal/api/errors_deployment.go index 70435254a..0da81d951 100644 --- a/services/core/internal/api/errors_deployment.go +++ b/services/core/internal/api/errors_deployment.go @@ -102,8 +102,6 @@ func writeSandboxError(w http.ResponseWriter, err error) bool { writeError(w, http.StatusUnauthorized, "invalid_node_credential", "A valid sandbox node enrollment or node credential is required.") case errors.Is(err, deployment.ErrNodeInUse): writeError(w, http.StatusConflict, "runtime_node_in_use", "The sandbox node retains allocations, snapshots, reservations or pending cleanup.") - case errors.Is(err, deployment.ErrLocalNodeConfigured): - writeError(w, http.StatusConflict, "runtime_local_node_configured", "The local sandbox node is enabled in deployment configuration. Drain it with the previous release and remove its file-managed configuration before replacing it.") case errors.Is(err, placement.ErrNodesPreparing): writeError(w, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime.") case errors.Is(err, placement.ErrNodeUnavailable): diff --git a/services/core/internal/api/errors_test.go b/services/core/internal/api/errors_test.go index 9f139d003..25cc78eb9 100644 --- a/services/core/internal/api/errors_test.go +++ b/services/core/internal/api/errors_test.go @@ -138,7 +138,6 @@ func TestConflictErrorsUseConflictType(t *testing.T) { for err, code := range map[error]string{ deployment.ErrConflict: "sandbox_deployment_conflict", deployment.ErrNodeInUse: "runtime_node_in_use", - deployment.ErrLocalNodeConfigured: "runtime_local_node_configured", deployment.ErrNodeAddressMismatch: "sandbox_node_address_mismatch", sessions.ErrEnvironmentUnavailable: "environment_unavailable", execution.ErrEnvironmentInputExpired: "environment_input_expired", diff --git a/services/core/internal/api/placement_errors_test.go b/services/core/internal/api/placement_errors_test.go index 90a8fb360..8aefad2be 100644 --- a/services/core/internal/api/placement_errors_test.go +++ b/services/core/internal/api/placement_errors_test.go @@ -19,7 +19,7 @@ func TestPlacementErrorsKeepTheirResponses(t *testing.T) { code, message string }{ {placement.ErrResetAdmission, http.StatusServiceUnavailable, "sandbox_reset_in_progress", "A sandbox reset is in progress."}, - {fmt.Errorf("%w: sandbox creation is paused for provider maintenance", placement.ErrAdmissionClosed), http.StatusConflict, "environment_unavailable", "The environment is no longer available for new input."}, + {fmt.Errorf("%w: sandbox installation does not match deployment", placement.ErrAdmissionClosed), http.StatusConflict, "environment_unavailable", "The environment is no longer available for new input."}, {placement.ErrPublicURLUnreachable, http.StatusConflict, "sandbox_configuration_error", placement.ErrPublicURLUnreachable.Error()}, {placement.ErrNodesPreparing, http.StatusServiceUnavailable, "sandbox_nodes_preparing", "Sandbox nodes are preparing the requested Runtime."}, {placement.ErrNodeUnavailable, http.StatusServiceUnavailable, "runtime_node_unavailable", "The selected sandbox node is unavailable or has no capacity."}, diff --git a/services/core/internal/api/sandbox_deployment_setup.go b/services/core/internal/api/sandbox_deployment_setup.go index b58295306..5a26510b8 100644 --- a/services/core/internal/api/sandbox_deployment_setup.go +++ b/services/core/internal/api/sandbox_deployment_setup.go @@ -51,7 +51,7 @@ type DeploymentReset interface { } // @Summary Initialize the deployment sandbox provider -// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; differing selections and file-managed deployments reject. This does not create compute or execute work. +// @Description Selects a provider, enforced resource limits and pinned Runtime release. Core derives the deployment's core_url from the installation public URL and rejects a core_url member with 400. E2B returns 409 sandbox_configuration_error while the public URL is loopback. E2B credentials are write-only. E2B may omit resources to adopt the validated template build's CPU and memory, returned in specification.resources. Requires explicit expected_generation, including zero at first setup. Stale retries reject before provider validation. An identical selection at the current generation is a no-op; a differing selection rejects. This does not create compute or execute work. // @Tags Sandbox Manager // @Produce json // @Security DeploymentAdminAuth diff --git a/services/core/internal/api/sandbox_manager_test.go b/services/core/internal/api/sandbox_manager_test.go index 7de004b17..6e7959622 100644 --- a/services/core/internal/api/sandbox_manager_test.go +++ b/services/core/internal/api/sandbox_manager_test.go @@ -69,15 +69,6 @@ func TestSandboxEnrollmentDoesNotAcceptProjectAsAdmin(t *testing.T) { } } -func TestSandboxLocalNodeRemovalExplainsDeploymentBinding(t *testing.T) { - request := httptest.NewRequest(http.MethodDelete, "/core/v1/sandbox/nodes/local", nil) - response := httptest.NewRecorder() - writeDeploymentError(response, request, deployment.ErrLocalNodeConfigured) - if response.Code != http.StatusConflict || !strings.Contains(response.Body.String(), "runtime_local_node_configured") || !strings.Contains(response.Body.String(), "previous release") { - t.Fatal(response.Code, response.Body.String()) - } -} - // nodeCapacity rejects max_active outside the deployment's node capacity // bounds. func nodeCapacity(active int) error { diff --git a/services/core/internal/db/queries/runtime_deployment.sql b/services/core/internal/db/queries/runtime_deployment.sql index 36f23dc1c..382d55726 100644 --- a/services/core/internal/db/queries/runtime_deployment.sql +++ b/services/core/internal/db/queries/runtime_deployment.sql @@ -1,10 +1,6 @@ -- name: LockRuntimeDeployment :one SELECT * FROM runtime_deployment WHERE singleton = true FOR UPDATE; --- name: SetRuntimeDeployment :exec -UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true; - -- name: CountRuntimeDeploymentResources :one SELECT (SELECT count(*) FROM runtime_allocations WHERE state <> 'released')::bigint AS allocations, diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index 403741dce..334617404 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -20,7 +20,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM sqlc.narg(node_id)::uuid AND p.released_at IS NULL - AND NOT (SELECT admission_paused FROM runtime_deployment) + AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > sqlc.arg(after_id)::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index 604a338e0..b6c45b2ce 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -1,6 +1,3 @@ --- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true; - -- name: GetRuntimeDeployment :one SELECT * FROM runtime_deployment WHERE singleton=true; diff --git a/services/core/internal/db/queries/runtime_suspension.sql b/services/core/internal/db/queries/runtime_suspension.sql index d4b6a6e21..6a1ccbeef 100644 --- a/services/core/internal/db/queries/runtime_suspension.sql +++ b/services/core/internal/db/queries/runtime_suspension.sql @@ -39,14 +39,6 @@ SELECT clock_timestamp()::timestamptz AS observed_at, FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id WHERE a.id = $1; --- name: CountRuntimeComputeReservations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended'; - --- name: CountRuntimeRetainedAllocations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released'; - -- name: RuntimeComputeBlocksAdmission :one SELECT EXISTS ( SELECT 1 FROM runtime_allocations a JOIN environments e ON e.id = a.environment_id diff --git a/services/core/internal/db/queries/sandbox_deployment_setup.sql b/services/core/internal/db/queries/sandbox_deployment_setup.sql index 142f0720f..09bd91540 100644 --- a/services/core/internal/db/queries/sandbox_deployment_setup.sql +++ b/services/core/internal/db/queries/sandbox_deployment_setup.sql @@ -1,5 +1,5 @@ -- name: ClaimWebSandboxDeployment :exec -UPDATE runtime_deployment SET installation_id=$1, web_managed=true, +UPDATE runtime_deployment SET installation_id=$1, owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true; -- name: InitializeSandboxDeployment :exec diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 3fad61493..0c2938bb1 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -1,6 +1,6 @@ -- name: StartSandboxReset :exec WITH clock AS MATERIALIZED (SELECT clock_timestamp() AS at) -UPDATE runtime_deployment SET admission_paused = true, reset_clear = sqlc.arg(clear), +UPDATE runtime_deployment SET reset_clear = sqlc.arg(clear), reset_requested_at = clock.at, reset_deadline_at = CASE WHEN sqlc.arg(clear)::text = 'auto' THEN clock.at + make_interval(secs => sqlc.arg(deadline_seconds)::int) END, @@ -14,7 +14,7 @@ UPDATE runtime_deployment SET reset_clear = 'force', reset_forced_at = clock.at, FROM clock WHERE singleton = true AND reset_clear = 'auto'; -- name: CancelSandboxReset :exec -UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, +UPDATE runtime_deployment SET reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true; @@ -24,7 +24,7 @@ UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode specification = '{}', idle_seconds = 0, retention_seconds = 0, provider_config = '{}'::jsonb, provider_metadata = '{}'::jsonb, provider_credential = NULL, generation = generation + 1, owner_epoch = owner_epoch + 1, - admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true; diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 134170554..048878096 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -239,12 +239,9 @@ type RuntimeDeployment struct { Singleton bool `json:"singleton"` InstallationID pgtype.UUID `json:"installation_id"` BackendFingerprint string `json:"backend_fingerprint"` - AdmissionPaused bool `json:"admission_paused"` UpdatedAt pgtype.Timestamptz `json:"updated_at"` ProviderKind string `json:"provider_kind"` - LocalNodeID pgtype.UUID `json:"local_node_id"` OwnerEpoch int64 `json:"owner_epoch"` - WebManaged bool `json:"web_managed"` IdleSeconds int64 `json:"idle_seconds"` RetentionSeconds int64 `json:"retention_seconds"` Generation int64 `json:"generation"` diff --git a/services/core/internal/db/sqlc/runtime_deployment.sql.go b/services/core/internal/db/sqlc/runtime_deployment.sql.go index e8f076e8b..15d71eef4 100644 --- a/services/core/internal/db/sqlc/runtime_deployment.sql.go +++ b/services/core/internal/db/sqlc/runtime_deployment.sql.go @@ -7,8 +7,6 @@ package sqlc import ( "context" - - "github.com/jackc/pgx/v5/pgtype" ) const countAddressBindings = `-- name: CountAddressBindings :one @@ -55,7 +53,7 @@ func (q *Queries) CountRuntimeDeploymentResources(ctx context.Context) (CountRun } const lockRuntimeDeployment = `-- name: LockRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true FOR UPDATE +SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true FOR UPDATE ` func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -65,12 +63,9 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, - &i.LocalNodeID, &i.OwnerEpoch, - &i.WebManaged, &i.IdleSeconds, &i.RetentionSeconds, &i.Generation, @@ -87,19 +82,3 @@ func (q *Queries) LockRuntimeDeployment(ctx context.Context) (RuntimeDeployment, ) return i, err } - -const setRuntimeDeployment = `-- name: SetRuntimeDeployment :exec -UPDATE runtime_deployment SET installation_id = $1, backend_fingerprint = $2, -admission_paused = $3, updated_at = clock_timestamp() WHERE singleton = true -` - -type SetRuntimeDeploymentParams struct { - InstallationID pgtype.UUID `json:"installation_id"` - BackendFingerprint string `json:"backend_fingerprint"` - AdmissionPaused bool `json:"admission_paused"` -} - -func (q *Queries) SetRuntimeDeployment(ctx context.Context, arg SetRuntimeDeploymentParams) error { - _, err := q.db.Exec(ctx, setRuntimeDeployment, arg.InstallationID, arg.BackendFingerprint, arg.AdmissionPaused) - return err -} diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index 6036aa233..e4bc7dae0 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -150,7 +150,7 @@ FROM environments e JOIN sessions s ON s.id=e.session_id LEFT JOIN runtime_placements p ON p.environment_id=e.id WHERE p.node_id IS NOT DISTINCT FROM $1::uuid AND p.released_at IS NULL - AND NOT (SELECT admission_paused FROM runtime_deployment) + AND (SELECT reset_clear IS NULL FROM runtime_deployment) AND e.id > $2::uuid AND s.deleted_at IS NULL AND e.status='pending' AND s.configuration->'environment'->>'type'='openai_hosted' AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id=e.id) diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index dc5d98088..e8ee97976 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -119,7 +119,7 @@ func (q *Queries) DisconnectRuntimeNode(ctx context.Context, arg DisconnectRunti } const getRuntimeDeployment = `-- name: GetRuntimeDeployment :one -SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton=true +SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton=true ` func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, error) { @@ -129,12 +129,9 @@ func (q *Queries) GetRuntimeDeployment(ctx context.Context) (RuntimeDeployment, &i.Singleton, &i.InstallationID, &i.BackendFingerprint, - &i.AdmissionPaused, &i.UpdatedAt, &i.ProviderKind, - &i.LocalNodeID, &i.OwnerEpoch, - &i.WebManaged, &i.IdleSeconds, &i.RetentionSeconds, &i.Generation, @@ -526,20 +523,6 @@ func (q *Queries) RemoveRuntimeNode(ctx context.Context, id pgtype.UUID) error { return err } -const setRuntimeManagerDeployment = `-- name: SetRuntimeManagerDeployment :exec -UPDATE runtime_deployment SET provider_kind=$1, local_node_id=$2, mode='nodes', generation=GREATEST(generation,1), owner_epoch=owner_epoch+1 WHERE singleton=true -` - -type SetRuntimeManagerDeploymentParams struct { - ProviderKind string `json:"provider_kind"` - LocalNodeID pgtype.UUID `json:"local_node_id"` -} - -func (q *Queries) SetRuntimeManagerDeployment(ctx context.Context, arg SetRuntimeManagerDeploymentParams) error { - _, err := q.db.Exec(ctx, setRuntimeManagerDeployment, arg.ProviderKind, arg.LocalNodeID) - return err -} - const setRuntimeObservation = `-- name: SetRuntimeObservation :exec UPDATE runtime_allocations SET observation_error=$4 WHERE id=$1 AND compute_revision=$2 AND state=$3 AND state<>'released' ` diff --git a/services/core/internal/db/sqlc/runtime_suspension.sql.go b/services/core/internal/db/sqlc/runtime_suspension.sql.go index 5c4867839..ed6b5bb50 100644 --- a/services/core/internal/db/sqlc/runtime_suspension.sql.go +++ b/services/core/internal/db/sqlc/runtime_suspension.sql.go @@ -27,30 +27,6 @@ func (q *Queries) ClearRuntimeWake(ctx context.Context, arg ClearRuntimeWakePara return err } -const countRuntimeComputeReservations = `-- name: CountRuntimeComputeReservations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' AND compute_phase <> 'suspended' -` - -func (q *Queries) CountRuntimeComputeReservations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { - row := q.db.QueryRow(ctx, countRuntimeComputeReservations, providerKey) - var count int64 - err := row.Scan(&count) - return count, err -} - -const countRuntimeRetainedAllocations = `-- name: CountRuntimeRetainedAllocations :one -SELECT count(*) FROM runtime_allocations -WHERE provider_key = $1 AND state <> 'released' -` - -func (q *Queries) CountRuntimeRetainedAllocations(ctx context.Context, providerKey pgtype.UUID) (int64, error) { - row := q.db.QueryRow(ctx, countRuntimeRetainedAllocations, providerKey) - var count int64 - err := row.Scan(&count) - return count, err -} - const getRuntimeActivity = `-- name: GetRuntimeActivity :one SELECT clock_timestamp()::timestamptz AS observed_at, GREATEST(a.compute_activity_at, diff --git a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go index 48cb02cc5..de2ee8d6f 100644 --- a/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go +++ b/services/core/internal/db/sqlc/sandbox_deployment_setup.sql.go @@ -21,7 +21,7 @@ func (q *Queries) AdvanceSandboxOwnerEpoch(ctx context.Context) error { } const claimWebSandboxDeployment = `-- name: ClaimWebSandboxDeployment :exec -UPDATE runtime_deployment SET installation_id=$1, web_managed=true, +UPDATE runtime_deployment SET installation_id=$1, owner_epoch=owner_epoch+1, updated_at=clock_timestamp() WHERE singleton=true ` diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 49512c3f6..3791f0e5d 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -12,7 +12,7 @@ import ( ) const cancelSandboxReset = `-- name: CancelSandboxReset :exec -UPDATE runtime_deployment SET admission_paused = false, reset_clear = NULL, +UPDATE runtime_deployment SET reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true @@ -28,7 +28,7 @@ UPDATE runtime_deployment SET provider_kind = '', backend_fingerprint = '', mode specification = '{}', idle_seconds = 0, retention_seconds = 0, provider_config = '{}'::jsonb, provider_metadata = '{}'::jsonb, provider_credential = NULL, generation = generation + 1, owner_epoch = owner_epoch + 1, - admission_paused = false, reset_clear = NULL, reset_requested_at = NULL, + reset_clear = NULL, reset_requested_at = NULL, reset_deadline_at = NULL, reset_forced_at = NULL, reset_audit = NULL, updated_at = clock_timestamp() WHERE singleton = true @@ -51,7 +51,7 @@ func (q *Queries) ForceSandboxReset(ctx context.Context) error { } const getSandboxDeploymentSnapshot = `-- name: GetSandboxDeploymentSnapshot :one -WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, admission_paused, updated_at, provider_kind, local_node_id, owner_epoch, web_managed, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true LIMIT 1), +WITH deployment AS MATERIALIZED (SELECT singleton, installation_id, backend_fingerprint, updated_at, provider_kind, owner_epoch, idle_seconds, retention_seconds, generation, mode, provider_credential, specification, reset_clear, reset_requested_at, reset_deadline_at, reset_forced_at, reset_audit, provider_config, provider_metadata FROM runtime_deployment WHERE singleton = true LIMIT 1), observed AS MATERIALIZED (SELECT clock_timestamp() AS as_of), held AS ( SELECT a.deployment_generation, a.node_id, s.id AS session_id, e.id AS environment_id, false AS pending, @@ -89,7 +89,7 @@ held AS ( ), offline AS ( SELECT node_id, name, count(*)::bigint AS resources FROM classified WHERE offline GROUP BY node_id, name ) -SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.admission_paused, d.updated_at, d.provider_kind, d.local_node_id, d.owner_epoch, d.web_managed, d.idle_seconds, d.retention_seconds, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, +SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.updated_at, d.provider_kind, d.owner_epoch, d.idle_seconds, d.retention_seconds, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, jsonb_build_object( @@ -128,12 +128,9 @@ func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxD &i.RuntimeDeployment.Singleton, &i.RuntimeDeployment.InstallationID, &i.RuntimeDeployment.BackendFingerprint, - &i.RuntimeDeployment.AdmissionPaused, &i.RuntimeDeployment.UpdatedAt, &i.RuntimeDeployment.ProviderKind, - &i.RuntimeDeployment.LocalNodeID, &i.RuntimeDeployment.OwnerEpoch, - &i.RuntimeDeployment.WebManaged, &i.RuntimeDeployment.IdleSeconds, &i.RuntimeDeployment.RetentionSeconds, &i.RuntimeDeployment.Generation, @@ -230,7 +227,7 @@ func (q *Queries) SessionBlocksAutoReset(ctx context.Context, sessionID pgtype.U const startSandboxReset = `-- name: StartSandboxReset :exec WITH clock AS MATERIALIZED (SELECT clock_timestamp() AS at) -UPDATE runtime_deployment SET admission_paused = true, reset_clear = $1, +UPDATE runtime_deployment SET reset_clear = $1, reset_requested_at = clock.at, reset_deadline_at = CASE WHEN $1::text = 'auto' THEN clock.at + make_interval(secs => $2::int) END, diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index cdb44204d..14be17800 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -322,14 +322,14 @@ func TestReserveAllocationAdmitsAndTakesTheReservedNode(t *testing.T) { findAllocation: func() (Allocation, bool, error) { return Allocation{}, false, nil }, lockDeployment: func() (placement.Deployment, error) { return deployment, nil }} } - paused := fresh() - paused.lockDeployment = func() (placement.Deployment, error) { + resetting := fresh() + resetting.lockDeployment = func() (placement.Deployment, error) { d := deployment - d.AdmissionPaused = true + d.Resetting = true return d, nil } - if _, err := allocationOperations(t, paused, sessions.LockedSession{}, nil).ReserveAllocation(t.Context(), key, installation, testCredentialHash()); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("paused admission reserved an allocation", err) + if _, err := allocationOperations(t, resetting, sessions.LockedSession{}, nil).ReserveAllocation(t.Context(), key, installation, testCredentialHash()); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("a resetting deployment reserved an allocation", err) } released := fresh() released.loadReserved = func() (placement.Reserved, error) { diff --git a/services/core/internal/deployment/errors.go b/services/core/internal/deployment/errors.go index 1875e9716..ff90ef7b4 100644 --- a/services/core/internal/deployment/errors.go +++ b/services/core/internal/deployment/errors.go @@ -20,7 +20,6 @@ var ( ErrNotConfigured = errors.New("the sandbox deployment is not configured") ErrNodeInUse = errors.New("sandbox node retains resources") ErrNodeCredential = errors.New("invalid sandbox node credential") - ErrLocalNodeConfigured = errors.New("local sandbox node is enabled in deployment configuration") // ErrAllocationConflict rejects an allocation change whose owner no longer // matches the stored allocation, device binding, state or compute revision, // or a replay for another installation. diff --git a/services/core/internal/deployment/execution.go b/services/core/internal/deployment/execution.go index 1b69cee68..0c1ef29fa 100644 --- a/services/core/internal/deployment/execution.go +++ b/services/core/internal/deployment/execution.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "math" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" @@ -41,7 +40,7 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) return err } if d.InstallationID != "" { - if !d.WebManaged || d.InstallationID != id { + if d.InstallationID != id { return ErrConflict } } else { @@ -62,125 +61,19 @@ func (e *ExecutionOperations) Claim(ctx context.Context, installationID string) }) } -// ConfigureProcess records the deployment process configuration selects, -// before the Worker starts. AdmissionPaused must be committed for the old -// installation before any switch. A nil selection never forgets the previous -// identity or unresolved resources. -func (e *ExecutionOperations) ConfigureProcess(ctx context.Context, selected *ProcessDeployment) error { - var installation string - if selected != nil { - copy := *selected - selected = © - id, err := parseID(selected.InstallationID) - if err != nil { - return err - } - installation = id - if !validDigest(selected.BackendFingerprint) { - return fmt.Errorf("%w: invalid backend identity fingerprint", ErrInvalidInput) - } - } +// RequireUnclaimed lets an execution owner without sandbox runtimes start +// only on a deployment no installation has claimed. +func (e *ExecutionOperations) RequireUnclaimed(ctx context.Context) error { return e.storage.WithDeployment(ctx, func(tx DeploymentTx) error { - previous, err := tx.LoadDeployment() + d, err := tx.LoadDeployment() if err != nil { return err } - if previous.WebManaged { + if d.InstallationID != "" { return ErrConflict } - if selected != nil && previous.InstallationID == installation && previous.BackendFingerprint == selected.BackendFingerprint && (previous.Provider == "" || selected.ProviderKind == previous.Provider) { - if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { - return err - } - return e.configureManager(tx, previous, selected, installation) - } - resources, err := tx.CountResources() - if err != nil { - return err - } - if selected == nil { - if previous.InstallationID != "" && (resources.Allocations != 0 || resources.Pending != 0) { - return fmt.Errorf("cannot disable managed sandbox provider: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - return nil - } - if previous.InstallationID == "" { - if resources.Allocations != 0 { - return fmt.Errorf("cannot adopt sandbox installation: %d existing unreleased allocations (including retained snapshots and pending cleanup) have no verified backend identity", resources.Allocations) - } - } else { - if !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("cannot switch sandbox installation: persist maintenance on the previous installation and keep the new installation in maintenance") - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot switch sandbox installation: %d unreleased allocations (instances, retained snapshots, uncertain operations or pending cleanup) and %d pending hosted environments remain", resources.Allocations, resources.Pending) - } - } - if err := tx.SetProcessDeployment(installation, selected.BackendFingerprint, selected.AdmissionPaused); err != nil { - return err - } - return e.configureManager(tx, previous, selected, installation) - }) -} - -// configureManager records the node provider and the local node process -// configuration selects. -func (e *ExecutionOperations) configureManager(tx DeploymentTx, previous Record, selected *ProcessDeployment, installation string) error { - if selected.ProviderKind == "" { return nil - } - isNode, err := e.service.registry.IsNode(selected.ProviderKind) - if err != nil { - return err - } - if !isNode { - return ErrInvalidInput - } - if previous.Provider == "" { - resources, err := tx.CountResources() - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 { - return fmt.Errorf("cannot adopt historical sandbox resources: keep the original Core responsible for retained resources and install this release separately") - } - } - var localNode string - if selected.LocalNodeID != "" { - id, err := parseID(selected.LocalNodeID) - if err != nil { - return err - } - localNode = id - if previous.LocalNodeID != "" && previous.LocalNodeID != id { - resources, err := tx.CountResources() - if err != nil { - return err - } - if resources.Allocations != 0 || resources.Pending != 0 || !previous.AdmissionPaused || !selected.AdmissionPaused { - return fmt.Errorf("local sandbox node identity changed: restore its original state directory; replacement requires maintenance and no retained resources") - } - } - if !validDigest(selected.LocalCredentialSHA256) { - return ErrInvalidInput - } - if err := validateNode("Local", selected.LocalMaxActive, selected.LocalMaxRetained); err != nil { - return err - } - n, err := tx.LoadNode(id) - if errors.Is(err, ErrNotFound) { - _, err = tx.InsertNode(NewNode{ID: id, InstallationID: installation, Name: "Local", BackendFingerprint: selected.BackendFingerprint, CredentialDigest: selected.LocalCredentialSHA256, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) - } else if err == nil { - if n.InstallationID != installation || n.BackendFingerprint != selected.BackendFingerprint || n.CredentialDigest != selected.LocalCredentialSHA256 { - return fmt.Errorf("local sandbox node identity does not match the retained backend") - } - err = tx.UpdateNode(id, NodeLimits{Name: n.Name, MaxActive: selected.LocalMaxActive, MaxRetained: selected.LocalMaxRetained}) - } - if err != nil { - return err - } - } - return tx.SetManagerDeployment(selected.ProviderKind, localNode) + }) } // CheckSetup rejects a stale or reset deployment before provider preparation. diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 526945814..3ccdb1d13 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -76,29 +76,27 @@ func (f *fakeExecutionStorage) WithDeployment(ctx context.Context, apply func(De } type fakeReader struct { - t testing.TB - deployment func(context.Context) (Record, error) - snapshot func(context.Context) (Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) - generations func(context.Context, int64) ([]GenerationRecord, error) - nodes func(context.Context) ([]NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) - readNodes func(context.Context, func(NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]ResetSession, error) - addressBindings func(context.Context, string) (AddressBindings, error) - environmentAllocation func(context.Context, AllocationKey) (Allocation, error) - credentialAllocations func(context.Context, string) ([]Allocation, error) - observationSessions func(context.Context, string, int) (ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) - activity func(context.Context, string) (Activity, error) - countComputeReservations func(context.Context, string) (int64, error) - countRetainedAllocations func(context.Context, string) (int64, error) + t testing.TB + deployment func(context.Context) (Record, error) + snapshot func(context.Context) (Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (AllocationRecord, error) + generations func(context.Context, int64) ([]GenerationRecord, error) + nodes func(context.Context) ([]NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (NodeRecord, []HostHistoryPoint, error) + readNodes func(context.Context, func(NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]ResetSession, error) + addressBindings func(context.Context, string) (AddressBindings, error) + environmentAllocation func(context.Context, AllocationKey) (Allocation, error) + credentialAllocations func(context.Context, string) ([]Allocation, error) + observationSessions func(context.Context, string, int) (ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, AllocationKey) (LifecyclePlacement, error) + activity func(context.Context, string) (Activity, error) } func (f *fakeReader) Deployment(ctx context.Context) (Record, error) { @@ -353,11 +351,6 @@ type fakeDeploymentTx struct { loadSnapshot func() (Snapshot, error) countResources func() (Resources, error) claimInstallation func(string) error - setProcessDeployment func(string, string, bool) error - setManagerDeployment func(string, string) error - loadNode func(string) (StoredNode, error) - insertNode func(NewNode) (StoredNode, error) - updateNode func(string, NodeLimits) error saveSelection func(SelectionRecord) error recordConfigurationMetadata func(json.RawMessage) error retainGeneration func() error @@ -399,41 +392,6 @@ func (f *fakeDeploymentTx) ClaimInstallation(installationID string) error { return f.claimInstallation(installationID) } -func (f *fakeDeploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { - if f.setProcessDeployment == nil { - unexpected(f.t, "SetProcessDeployment") - } - return f.setProcessDeployment(installationID, backendFingerprint, admissionPaused) -} - -func (f *fakeDeploymentTx) SetManagerDeployment(provider, localNodeID string) error { - if f.setManagerDeployment == nil { - unexpected(f.t, "SetManagerDeployment") - } - return f.setManagerDeployment(provider, localNodeID) -} - -func (f *fakeDeploymentTx) LoadNode(id string) (StoredNode, error) { - if f.loadNode == nil { - unexpected(f.t, "LoadNode") - } - return f.loadNode(id) -} - -func (f *fakeDeploymentTx) InsertNode(node NewNode) (StoredNode, error) { - if f.insertNode == nil { - unexpected(f.t, "InsertNode") - } - return f.insertNode(node) -} - -func (f *fakeDeploymentTx) UpdateNode(id string, limits NodeLimits) error { - if f.updateNode == nil { - unexpected(f.t, "UpdateNode") - } - return f.updateNode(id, limits) -} - func (f *fakeDeploymentTx) SaveSelection(selection SelectionRecord) error { if f.saveSelection == nil { unexpected(f.t, "SaveSelection") @@ -623,20 +581,6 @@ func (f *fakeReader) Activity(ctx context.Context, allocationID string) (Activit return f.activity(ctx, allocationID) } -func (f *fakeReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - if f.countComputeReservations == nil { - unexpected(f.t, "CountComputeReservations") - } - return f.countComputeReservations(ctx, installationID) -} - -func (f *fakeReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - if f.countRetainedAllocations == nil { - unexpected(f.t, "CountRetainedAllocations") - } - return f.countRetainedAllocations(ctx, installationID) -} - // fakeSessionReader serves the Session reads the observation resolver makes; // every other read fails the test. type fakeSessionReader struct { diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index 3f2ab3764..91867c15f 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -133,9 +133,6 @@ func (s *Service) RemoveNode(ctx context.Context, id string) error { if n.Retained != 0 || n.CleanupPending != 0 { return ErrNodeInUse } - if d.LocalNodeID == nodeID { - return ErrLocalNodeConfigured - } return tx.RemoveNode(nodeID) } return ErrNotFound @@ -165,7 +162,7 @@ func (s *Service) CreateEnrollment(ctx context.Context, capacity Capacity) (Enro if d.Reset != nil { return ErrResetInProgress } - if d.Mode != "nodes" || d.AdmissionPaused { + if d.Mode != "nodes" { return ErrConflict } if _, err := s.specification(d); err != nil { @@ -214,7 +211,7 @@ func (s *Service) Enroll(ctx context.Context, token string, input Enrollment) (N if d.Reset != nil { return ErrResetInProgress } - if d.Mode != "nodes" || d.AdmissionPaused || input.Provider != d.Provider { + if d.Mode != "nodes" || input.Provider != d.Provider { return ErrInvalidInput } spec, err := s.specification(d) @@ -426,9 +423,6 @@ func (s *Service) NodeConfiguration(ctx context.Context, nodeID, token string, g if err != nil { return err } - if node == nil && d.AdmissionPaused { - return ErrConflict - } limit, err := s.registry.RetainedLimit(d.Provider, active, retained) if err != nil { return err diff --git a/services/core/internal/deployment/placement/placement.go b/services/core/internal/deployment/placement/placement.go index 433e6f6b8..9b0c4277a 100644 --- a/services/core/internal/deployment/placement/placement.go +++ b/services/core/internal/deployment/placement/placement.go @@ -20,8 +20,7 @@ var ( // admission. ErrResetAdmission = errors.New("hosted admission is paused for a sandbox reset") // ErrAdmissionClosed rejects new hosted work that the deployment cannot - // admit: paused for maintenance, without a valid specification, or for - // another installation. + // admit: without a valid specification, or for another installation. ErrAdmissionClosed = errors.New("environment is no longer available") // ErrPublicURLUnreachable rejects selection and admission when the provider // requires a reachable public origin and the installation is loopback. @@ -67,12 +66,10 @@ func (r *Rules) PublicURL() string { return r.publicURL } // Deployment is the deployment as placement reads it, loaded under the // deployment lock. type Deployment struct { - // InstallationID is empty until an installation is claimed or configured. - InstallationID string - Provider, Mode string - Generation uint64 - WebManaged bool - AdmissionPaused bool + // InstallationID is empty until Web setup claims an installation. + InstallationID string + Provider, Mode string + Generation uint64 // Resetting reports a sandbox reset in progress. Resetting bool Specification json.RawMessage @@ -139,9 +136,6 @@ func (r *Rules) CheckAdmission(d Deployment, installation string) error { if d.Resetting { return ErrResetAdmission } - if d.AdmissionPaused { - return fmt.Errorf("%w: sandbox creation is paused for provider maintenance", ErrAdmissionClosed) - } if d.Provider != "" { var spec sandbox.DeploymentSpec if json.Unmarshal(d.Specification, &spec) != nil || r.declarations.ValidateSpecification(d.Provider, spec) != nil { @@ -157,8 +151,8 @@ func (r *Rules) CheckAdmission(d Deployment, installation string) error { // DecidePlacement chooses the node a new Session's hosted Environment // reserves, or nil when the deployment places no node: in direct mode and // without a provider. It prefers the highest ready generation, then the -// fewest active sandboxes. A Web-managed installation places only on nodes -// enrolled with its public URL; restores still reach the others. +// fewest active sandboxes. It places only on nodes enrolled with the public +// URL; restores still reach the others. func (r *Rules) DecidePlacement(d Deployment, nodes []Node) (*Placement, error) { if d.Resetting { return nil, ErrResetAdmission @@ -171,26 +165,20 @@ func (r *Rules) DecidePlacement(d Deployment, nodes []Node) (*Placement, error) } } if d.Mode == "direct" { - if d.AdmissionPaused { - return nil, ErrNodeUnavailable - } return nil, nil } if d.Provider == "" { - if d.WebManaged { + if d.InstallationID != "" { return nil, ErrNodeUnavailable } return nil, nil } - if d.AdmissionPaused { - return nil, ErrNodeUnavailable - } var chosen *Node preparing := false for i := range nodes { n := &nodes[i] free := n.Active < int64(n.MaxActive) && n.Retained < int64(n.MaxRetained) - reachable := !d.WebManaged || n.CoreURL == r.publicURL + reachable := n.CoreURL == r.publicURL if n.Online && n.TargetState == "preparing" && free && reachable { preparing = true } diff --git a/services/core/internal/deployment/placement/placement_test.go b/services/core/internal/deployment/placement/placement_test.go index ff7212cde..e60242f4d 100644 --- a/services/core/internal/deployment/placement/placement_test.go +++ b/services/core/internal/deployment/placement/placement_test.go @@ -93,11 +93,10 @@ func TestCheckAdmission(t *testing.T) { want error message string }{ - "unclaimed admits everything": {Deployment{Resetting: true, AdmissionPaused: true}, installation, nil, ""}, + "unclaimed admits everything": {Deployment{Resetting: true}, installation, nil, ""}, "admitted": {valid, installation, nil, ""}, "new Session": {valid, "", nil, ""}, - "reset": {with(func(d *Deployment) { d.Resetting, d.AdmissionPaused = true, true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, - "paused": {with(func(d *Deployment) { d.AdmissionPaused = true }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation is paused for provider maintenance"}, + "reset": {with(func(d *Deployment) { d.Resetting = true }), installation, ErrResetAdmission, "hosted admission is paused for a sandbox reset"}, "malformed specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`[`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, "rejected specification": {with(func(d *Deployment) { d.Specification = json.RawMessage(`{"resources":{"cpus":3}}`) }), installation, ErrAdmissionClosed, "environment is no longer available: sandbox creation requires a deployment specification"}, "no provider": {with(func(d *Deployment) { d.Provider, d.Specification = "", json.RawMessage(`[`) }), installation, nil, ""}, @@ -117,7 +116,7 @@ func TestDecidePlacement(t *testing.T) { ready := func(id string, g uint64, active int64) Node { return Node{ID: id, Online: true, ServingReady: true, ReadyGeneration: generation(g), Active: active, MaxActive: 4, Retained: active, MaxRetained: 4, CoreURL: publicURL} } - nodes := Deployment{Provider: "docker", Mode: "nodes", WebManaged: true} + nodes := Deployment{InstallationID: "installation", Provider: "docker", Mode: "nodes"} with := func(change func(*Deployment)) Deployment { d := nodes change(&d) @@ -140,16 +139,13 @@ func TestDecidePlacement(t *testing.T) { "reset": {origin(false), publicURL, with(func(d *Deployment) { d.Resetting = true }), nil, nil, ErrResetAdmission}, "loopback public origin": {origin(true), "http://localhost:8091", nodes, []Node{ready("a", 1, 0)}, nil, ErrPublicURLUnreachable}, "direct": {origin(false), publicURL, with(func(d *Deployment) { d.Mode = "direct" }), nil, nil, nil}, - "direct paused": {origin(false), publicURL, with(func(d *Deployment) { d.Mode, d.AdmissionPaused = "direct", true }), nil, nil, ErrNodeUnavailable}, "no provider": {&fakeDeclarations{t: t}, publicURL, Deployment{}, nil, nil, nil}, - "no provider on Web": {&fakeDeclarations{t: t}, publicURL, Deployment{WebManaged: true}, nil, nil, ErrNodeUnavailable}, - "paused": {origin(false), publicURL, with(func(d *Deployment) { d.AdmissionPaused = true }), []Node{ready("a", 1, 0)}, nil, ErrNodeUnavailable}, + "no provider on Web": {&fakeDeclarations{t: t}, publicURL, Deployment{InstallationID: "installation"}, nil, nil, ErrNodeUnavailable}, "no nodes": {origin(false), publicURL, nodes, nil, nil, ErrNodeUnavailable}, "only ineligible nodes": {origin(false), publicURL, nodes, []Node{offline, unready, full, retainedFull, elsewhere, {ID: "never", Online: true, ServingReady: true, MaxActive: 1, MaxRetained: 1, CoreURL: publicURL}}, nil, ErrNodeUnavailable}, "preparing": {origin(false), publicURL, nodes, []Node{offline, preparing}, nil, ErrNodesPreparing}, "highest generation": {origin(false), publicURL, nodes, []Node{ready("old", 1, 0), ready("new", 2, 3), preparing}, &Placement{NodeID: "new", Generation: 2}, nil}, "fewest active": {origin(false), publicURL, nodes, []Node{ready("busy", 2, 3), ready("idle", 2, 1)}, &Placement{NodeID: "idle", Generation: 2}, nil}, - "other address off Web": {origin(false), publicURL, with(func(d *Deployment) { d.WebManaged = false }), []Node{elsewhere}, &Placement{NodeID: "elsewhere", Generation: 9}, nil}, "public origin on public URL": {origin(true), publicURL, nodes, []Node{ready("a", 1, 0)}, &Placement{NodeID: "a", Generation: 1}, nil}, } { got, err := rules(t, test.declarations, test.url).DecidePlacement(test.d, test.nodes) diff --git a/services/core/internal/deployment/rules.go b/services/core/internal/deployment/rules.go index 91717deb0..d4ab12b35 100644 --- a/services/core/internal/deployment/rules.go +++ b/services/core/internal/deployment/rules.go @@ -70,7 +70,7 @@ func checkGeneration(d Record, installation string, generation uint64) error { if d.Generation != generation { return &GenerationStaleError{CurrentGeneration: d.Generation} } - if !d.WebManaged || d.InstallationID != installation { + if d.InstallationID == "" || d.InstallationID != installation { return ErrConflict } return nil diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go index ea2926e65..1c4819293 100644 --- a/services/core/internal/deployment/rules_test.go +++ b/services/core/internal/deployment/rules_test.go @@ -121,7 +121,7 @@ func TestValidateNode(t *testing.T) { func TestCheckGeneration(t *testing.T) { installation := uuid.NewString() - current := Record{InstallationID: installation, WebManaged: true, Generation: 3} + current := Record{InstallationID: installation, Generation: 3} if err := checkGeneration(current, installation, 3); err != nil { t.Fatal(err) } @@ -129,12 +129,12 @@ func TestCheckGeneration(t *testing.T) { if err := checkGeneration(current, installation, 2); !errors.As(err, &stale) || stale.CurrentGeneration != 3 || !errors.Is(err, ErrConflict) { t.Fatalf("stale generation: %v", err) } - process := current - process.WebManaged = false + unclaimed := current + unclaimed.InstallationID = "" for _, c := range []struct { d Record installation string - }{{process, installation}, {current, uuid.NewString()}} { + }{{unclaimed, installation}, {current, uuid.NewString()}} { if err := checkGeneration(c.d, c.installation, 3); !errors.Is(err, ErrConflict) || errors.As(err, &stale) { t.Errorf("checkGeneration(%+v, %s) = %v, want a plain conflict", c.d, c.installation, err) } diff --git a/services/core/internal/deployment/service.go b/services/core/internal/deployment/service.go index 673a41dac..575f0d79f 100644 --- a/services/core/internal/deployment/service.go +++ b/services/core/internal/deployment/service.go @@ -88,10 +88,10 @@ func (s *Service) Setup(ctx context.Context) (Setup, error) { } func (s *Service) setup(d Record) (Setup, error) { - if !d.WebManaged { + if d.InstallationID == "" { return Setup{}, ErrConflict } - result := Setup{InstallationID: d.InstallationID, Provider: d.Provider, BackendFingerprint: d.BackendFingerprint, Generation: d.Generation, Mode: d.Mode, AdmissionPaused: d.AdmissionPaused} + result := Setup{InstallationID: d.InstallationID, Provider: d.Provider, BackendFingerprint: d.BackendFingerprint, Generation: d.Generation, Mode: d.Mode} if err := json.Unmarshal(d.Specification, &result.Specification); err != nil { return Setup{}, err } diff --git a/services/core/internal/deployment/service_test.go b/services/core/internal/deployment/service_test.go index 9fc5964aa..e3554720f 100644 --- a/services/core/internal/deployment/service_test.go +++ b/services/core/internal/deployment/service_test.go @@ -71,7 +71,7 @@ func webDeployment(t *testing.T, installation, provider string, generation uint6 if err != nil { t.Fatal(err) } - return Record{InstallationID: installation, WebManaged: true, Provider: provider, Generation: generation, Mode: "nodes", Specification: specification, + return Record{InstallationID: installation, Provider: provider, Generation: generation, Mode: "nodes", Specification: specification, Configuration: sandbox.ConfigurationRecord{Public: json.RawMessage(`{}`), Metadata: json.RawMessage(`{}`)}} } @@ -178,15 +178,6 @@ func TestRegistryLookupFailuresPropagate(t *testing.T) { if _, err := service.ListNodes(t.Context()); !errors.Is(err, providers.ErrUnknownProvider) { t.Errorf("ListNodes = %v", err) } - tx := &fakeDeploymentTx{t: t, - loadDeployment: func() (Record, error) { return Record{}, nil }, - countResources: func() (Resources, error) { return Resources{}, nil }, - setProcessDeployment: func(string, string, bool) error { return nil }, - } - process := &ProcessDeployment{ProviderKind: "retired", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64)} - if err := operations(t, testPublicURL, tx).ConfigureProcess(t.Context(), process); !errors.Is(err, providers.ErrUnknownProvider) { - t.Errorf("ConfigureProcess = %v", err) - } } // Setup carries the mode and operations the provider declares. A stored @@ -280,8 +271,6 @@ func TestEnrollChecksTheTokenBeforeTheDeployment(t *testing.T) { receipt := EnrollmentRecord{ID: uuid.NewString(), InstallationID: installation, ExpiresAt: time.Now().Add(10 * time.Minute), MaxActive: 1, MaxRetained: 1} resetting := current resetting.Reset = &ResetState{Clear: "sandboxes", RequestedAt: time.Now()} - paused := current - paused.AdmissionPaused = true unselected := current unselected.Provider = "" consumed, expired, foreign := receipt, receipt, receipt @@ -303,12 +292,11 @@ func TestEnrollChecksTheTokenBeforeTheDeployment(t *testing.T) { }{ {"unknown token while resetting", resetting, EnrollmentRecord{}, ErrNotFound, valid, ErrNodeCredential}, {"consumed token before setup", Record{}, consumed, nil, valid, ErrNodeCredential}, - {"expired token while paused", paused, expired, nil, valid, ErrNodeCredential}, + {"expired token while resetting", resetting, expired, nil, valid, ErrNodeCredential}, {"token of another installation while resetting", resetting, foreign, nil, valid, ErrNodeCredential}, {"token store failure", current, EnrollmentRecord{}, errors.New("database down"), valid, placement.ErrNodeUnavailable}, {"no provider selected", unselected, receipt, nil, valid, placement.ErrNodeUnavailable}, {"reset in progress", resetting, receipt, nil, valid, ErrResetInProgress}, - {"admission paused", paused, receipt, nil, valid, ErrInvalidInput}, {"stale generation", current, receipt, nil, stale, ErrSpecificationMismatch}, // The token stays unused: the fake allows no insert or consumption. {"another Core address", current, receipt, nil, elsewhere, ErrNodeAddressMismatch}, diff --git a/services/core/internal/deployment/session_archive.go b/services/core/internal/deployment/session_archive.go index d2b2578fd..eb33b012d 100644 --- a/services/core/internal/deployment/session_archive.go +++ b/services/core/internal/deployment/session_archive.go @@ -119,13 +119,13 @@ func (e *ExecutionOperations) archiveSession(ctx context.Context, tenantID, sess } // checkArchiveDeployment rejects an archive against a deployment whose -// generation is not the expected one, that Web does not manage or that has no +// generation is not the expected one, that has no installation or that has no // provider. func checkArchiveDeployment(d Record, expectedGeneration uint64) error { if d.Generation != expectedGeneration { return &GenerationStaleError{CurrentGeneration: d.Generation} } - if !d.WebManaged || d.InstallationID == "" { + if d.InstallationID == "" { return ErrConflict } if d.Provider == "" { diff --git a/services/core/internal/deployment/session_archive_test.go b/services/core/internal/deployment/session_archive_test.go index 3ebbd7cdb..2db8c361c 100644 --- a/services/core/internal/deployment/session_archive_test.go +++ b/services/core/internal/deployment/session_archive_test.go @@ -14,14 +14,13 @@ import ( ) func TestCheckArchiveDeployment(t *testing.T) { - managed := Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 4} + managed := Record{InstallationID: "installation", Provider: "docker", Generation: 4} for name, test := range map[string]struct { change func(*Record) want error }{ "current": {func(*Record) {}, nil}, - "stale generation": {func(d *Record) { d.Generation = 5; d.WebManaged = false }, &GenerationStaleError{CurrentGeneration: 5}}, - "process managed": {func(d *Record) { d.WebManaged = false; d.Provider = "" }, ErrConflict}, + "stale generation": {func(d *Record) { d.Generation = 5; d.InstallationID = "" }, &GenerationStaleError{CurrentGeneration: 5}}, "no installation": {func(d *Record) { d.InstallationID = "" }, ErrConflict}, "no provider": {func(d *Record) { d.Provider = "" }, ErrNotConfigured}, } { @@ -225,7 +224,7 @@ func archiveOperations(t *testing.T, tx *fakeArchiveTx, locked sessions.LockedSe } func TestArchiveSession(t *testing.T) { - managed := Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 1} + managed := Record{InstallationID: "installation", Provider: "docker", Generation: 1} hosted := &sessions.Environment{ID: "environment", Status: "connected", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} expired := &sessions.Environment{ID: "environment", Status: "expired", Configuration: hosted.Configuration} live := &Allocation{ID: "allocation", DeviceID: "device", ProviderKey: "installation", State: "running"} @@ -281,7 +280,7 @@ func TestArchiveSession(t *testing.T) { func TestArchiveResetSession(t *testing.T) { requested := time.Unix(100, 0) resetting := func(clear string) Record { - return Record{InstallationID: "installation", WebManaged: true, Provider: "docker", Generation: 1, Reset: &ResetState{Clear: clear, RequestedAt: requested}} + return Record{InstallationID: "installation", Provider: "docker", Generation: 1, Reset: &ResetState{Clear: clear, RequestedAt: requested}} } hosted := &sessions.Environment{ID: "environment", Status: "connected", Configuration: json.RawMessage(`{"type":"openai_hosted"}`)} failed := &sessions.Environment{ID: "environment", Status: "failed", Configuration: hosted.Configuration} diff --git a/services/core/internal/deployment/setup.go b/services/core/internal/deployment/setup.go index e40852e6c..2f17624b0 100644 --- a/services/core/internal/deployment/setup.go +++ b/services/core/internal/deployment/setup.go @@ -16,8 +16,7 @@ type Setup struct { Generation uint64 // Mode is where the provider runs: "nodes" for enrolled sandbox nodes and // "direct" for a provider Core calls itself. - Mode string - AdmissionPaused bool + Mode string // Operations is the provider's declared operation support, which the node // transport proxies. Operations providercontract.Operations @@ -35,19 +34,6 @@ func (s Setup) selection() sandbox.Selection { return sandbox.Selection{Provider: s.Provider, DeploymentSpec: s.Specification, Configuration: s.Configuration} } -// ProcessDeployment is a deployment selected by process configuration instead of -// Web setup. Its fingerprint describes the backend namespace, never credentials -// or image contents. -type ProcessDeployment struct { - ProviderKind string - LocalNodeID string - LocalCredentialSHA256 string - LocalMaxActive, LocalMaxRetained int - InstallationID string - BackendFingerprint string - AdmissionPaused bool -} - // configurationJSON reports an absent configuration object as {}. func configurationJSON(raw json.RawMessage) json.RawMessage { if len(raw) == 0 { diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index 20cff0768..e3b9dc761 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -249,12 +249,6 @@ type Reader interface { // Activity returns the allocation's activity; a missing allocation is // ErrNotFound. Activity(ctx context.Context, allocationID string) (Activity, error) - // CountComputeReservations counts the installation's allocations that - // reserve active compute. - CountComputeReservations(ctx context.Context, installationID string) (int64, error) - // CountRetainedAllocations counts the installation's allocations that - // retain resources. - CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) } // NodeReads loads node authentication facts. @@ -306,13 +300,6 @@ type DeploymentTx interface { // ClaimInstallation reserves the installation for Web setup and fences the // previous owner epoch's node presence. ClaimInstallation(installationID string) error - SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error - // SetManagerDeployment records the node provider and the local node, which - // is empty when there is none. - SetManagerDeployment(provider, localNodeID string) error - LoadNode(id string) (StoredNode, error) - InsertNode(node NewNode) (StoredNode, error) - UpdateNode(id string, limits NodeLimits) error // SaveSelection stores the next generation. It seals a secret bound to the // installation and generation; without a key it returns // credentialcrypto.ErrUnavailable. @@ -348,16 +335,13 @@ type DeploymentTx interface { // Record is the stored deployment. type Record struct { - // InstallationID is empty until an installation is claimed or configured. + // InstallationID is empty until Web setup claims an installation. InstallationID string - WebManaged bool Provider string BackendFingerprint string Generation uint64 OwnerEpoch uint64 Mode string - AdmissionPaused bool - LocalNodeID string IdleSeconds int64 RetentionSeconds int64 // Specification is the stored specification document. diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 94bacf49e..6562e189a 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -52,6 +52,15 @@ func unitDeploymentService(t *testing.T) *deployment.Service { return service } +// unusedPreparation is the preparer of a test that submits no sandbox +// selection; preparing one fails the test. +func unusedPreparation(t *testing.T) RuntimeDeploymentPreparer { + return func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) { + t.Error("the test prepared a sandbox selection it did not submit") + return PreparedRuntimeDeployment{}, errors.New("unexpected sandbox selection preparation") + } +} + // deploymentOperations builds the deployment service on storage and reader and // the execution operations on execution. func deploymentOperations(t *testing.T, storage deployment.Storage, reader deployment.Reader, execution deployment.ExecutionStorage) (*deployment.Service, *deployment.ExecutionOperations) { @@ -132,29 +141,27 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - t *testing.T - deployment func(context.Context) (deployment.Record, error) - snapshot func(context.Context) (deployment.Snapshot, error) - ownerEpoch func(context.Context) (uint64, error) - allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) - generations func(context.Context, int64) ([]deployment.GenerationRecord, error) - nodes func(context.Context) ([]deployment.NodeRecord, error) - nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) - readNodes func(context.Context, func(deployment.NodeReads) error) error - resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) - addressBindings func(context.Context, string) (deployment.AddressBindings, error) - environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) - credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) - observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) - nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) - nodeOnline func(context.Context, string) (bool, error) - lifecycleNodes func(context.Context) ([]string, error) - lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) - unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) - lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) - activity func(context.Context, string) (deployment.Activity, error) - countComputeReservations func(context.Context, string) (int64, error) - countRetainedAllocations func(context.Context, string) (int64, error) + t *testing.T + deployment func(context.Context) (deployment.Record, error) + snapshot func(context.Context) (deployment.Snapshot, error) + ownerEpoch func(context.Context) (uint64, error) + allocation func(context.Context, sandbox.Reference) (deployment.AllocationRecord, error) + generations func(context.Context, int64) ([]deployment.GenerationRecord, error) + nodes func(context.Context) ([]deployment.NodeRecord, error) + nodeHistory func(context.Context, string, coremetrics.Range) (deployment.NodeRecord, []deployment.HostHistoryPoint, error) + readNodes func(context.Context, func(deployment.NodeReads) error) error + resetSessions func(context.Context, string, bool) ([]deployment.ResetSession, error) + addressBindings func(context.Context, string) (deployment.AddressBindings, error) + environmentAllocation func(context.Context, deployment.AllocationKey) (deployment.Allocation, error) + credentialAllocations func(context.Context, string) ([]deployment.Allocation, error) + observationSessions func(context.Context, string, int) (deployment.ObservationSessionPage, error) + nodeAllocations func(context.Context, string) ([]deployment.NodeAllocation, error) + nodeOnline func(context.Context, string) (bool, error) + lifecycleNodes func(context.Context) ([]string, error) + lifecycleAllocations func(context.Context, string, string) ([]deployment.Allocation, error) + unallocatedEnvironments func(context.Context, string, string) ([]deployment.UnallocatedEnvironment, error) + lifecyclePlacement func(context.Context, deployment.AllocationKey) (deployment.LifecyclePlacement, error) + activity func(context.Context, string) (deployment.Activity, error) } func (r *strictDeploymentReader) Deployment(ctx context.Context) (deployment.Record, error) { @@ -335,17 +342,3 @@ func (r *strictDeploymentReader) Activity(ctx context.Context, allocationID stri } return r.activity(ctx, allocationID) } - -func (r *strictDeploymentReader) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - if r.countComputeReservations == nil { - return 0, unexpectedDeploymentCall(r.t, "CountComputeReservations") - } - return r.countComputeReservations(ctx, installationID) -} - -func (r *strictDeploymentReader) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - if r.countRetainedAllocations == nil { - return 0, unexpectedDeploymentCall(r.t, "CountRetainedAllocations") - } - return r.countRetainedAllocations(ctx, installationID) -} diff --git a/services/core/internal/execution/owner_test.go b/services/core/internal/execution/owner_test.go index 57d676ee9..9da8bdb4b 100644 --- a/services/core/internal/execution/owner_test.go +++ b/services/core/internal/execution/owner_test.go @@ -134,7 +134,7 @@ func TestStartWorkerFailureClosesLeaseOnce(t *testing.T) { lease.inner = owner.Lease id := uuid.NewString() service, reader := unusedSessions(t) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} _, err := StartWorker(canceled, dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if ping := owner.Lease.CheckOwnership(t.Context()); !errors.Is(ping, pgunit.ErrLeaseClosed) { t.Error("failed start kept the database lease", ping) @@ -193,7 +193,7 @@ func TestWorkerRunClosesLeaseAfterDrain(t *testing.T) { id := uuid.NewString() // The Worker's first reconciliation scans the Session work. reader, service := testSessions(t, pool, nil) - dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })} + dispatcher := &Dispatcher{Registry: runtimegateway.NewRegistry(), Credentials: &recordingCredentials{}, Observer: unusedObserver{t}, Deployment: deployments, DeploymentReader: deploymentReader, Sessions: service, SessionsReader: reader, ManagedRuntimes: NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))} worker, err := StartWorker(t.Context(), dispatcher, Owner{Lease: lease, Deployment: owner.Deployment, Sessions: owner.Sessions}) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index 7d08a1b33..b093ab975 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -12,13 +12,11 @@ import ( "github.com/google/uuid" ) -// RuntimeSuspensionPolicy applies only to an explicitly qualified single-host -// provider. Fixed guest sizing plus MaxActive bounds reserved CPU and memory. +// RuntimeSuspensionPolicy is the idle suspension policy of a provider that +// suspends sandboxes. type RuntimeSuspensionPolicy struct { IdleTimeout time.Duration Retention time.Duration - MaxActive int - MaxRetained int } type runtimeCompute struct { @@ -30,23 +28,6 @@ type runtimeCompute struct { Rollback bool `json:"rollback,omitempty"` } -func (r *runtimeLifecycle) computeCapacity(ctx context.Context, key string) error { - policy := r.config.Suspension - if key != r.config.InstallationID { - return sandbox.ErrOwnership - } - if policy == nil { - return nil - } - count, err := r.reader.CountComputeReservations(ctx, key) - if err != nil { - return err - } - if count >= int64(policy.MaxActive) { - return ErrExecutionUnavailable - } - return nil -} func (r *runtimeLifecycle) saveCompute(ctx context.Context, owner deployment.Allocation, phase string, state runtimeCompute, until *time.Time) (deployment.Allocation, error) { raw, err := json.Marshal(state) if err != nil { @@ -230,9 +211,6 @@ func (r *runtimeLifecycle) restoreIdleCompute(ctx context.Context, p sandbox.Che if !activity.Busy && !activity.WakeRequested { return nil } - if err := r.computeCapacityForAllocation(ctx, owner); err != nil { - return err - } if state.Snapshot == nil || state.Target != nil { return sandbox.ErrOwnership } @@ -274,11 +252,3 @@ func ignoreComputeAbsent(err error) error { } return err } - -// Node-backed restores reserve capacity atomically in SetCompute. -func (r *runtimeLifecycle) computeCapacityForAllocation(ctx context.Context, owner deployment.Allocation) error { - if owner.NodeID != "" { - return nil - } - return r.computeCapacity(ctx, owner.ProviderKey) -} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index d6ca1b27c..5e275b09e 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -23,21 +23,17 @@ import ( // BackendFingerprint identifies its namespace independently of mutable sizing. type RuntimeProvider struct { // PublishUnconfigured updates the shared observation cache after reset commit. - PublishUnconfigured func(uint64) - Generation uint64 - Mode string - loadDeployment func(context.Context) (*RuntimeProvider, error) - prepareDeployment RuntimeDeploymentPreparer - ProviderKind string - LocalNodeID string - LocalCredentialSHA256 string - LocalMaxActive, LocalMaxRetained int - CoreURL string - InstallationID string - BackendFingerprint string - Provider sandbox.SandboxProvider - AdmissionPaused bool - Suspension *RuntimeSuspensionPolicy + PublishUnconfigured func(uint64) + Generation uint64 + Mode string + loadDeployment func(context.Context) (*RuntimeProvider, error) + prepareDeployment RuntimeDeploymentPreparer + ProviderKind string + CoreURL string + InstallationID string + BackendFingerprint string + Provider sandbox.SandboxProvider + Suspension *RuntimeSuspensionPolicy } type runtimeLifecycle struct { @@ -67,21 +63,12 @@ func newRuntimeManager(owner Owner, deployments *deployment.Service, deploymentR if config == nil { return nil, nil } - var copied RuntimeProvider - if config.loadDeployment != nil { - id, err := uuid.Parse(config.InstallationID) - if err != nil || id == uuid.Nil || id.String() != config.InstallationID || registry == nil { - return nil, sandbox.ErrInvalid - } - } else { - var err error - copied, err = validatedRuntimeProvider(config, registry) - if err != nil { - return nil, err - } + id, err := uuid.Parse(config.InstallationID) + if err != nil || id == uuid.Nil || id.String() != config.InstallationID || config.loadDeployment == nil || config.prepareDeployment == nil || registry == nil { + return nil, sandbox.ErrInvalid } ctx, stop := context.WithCancel(context.Background()) - return &runtimeManager{sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, config: copied, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil + return &runtimeManager{sessions: sessionReader, sessionExecution: owner.Sessions, deployment: owner.Deployment, deploymentService: deployments, deploymentReader: deploymentReader, lease: owner.Lease, registry: registry, setupInstallationID: config.InstallationID, loadDeployment: config.loadDeployment, prepareDeployment: config.prepareDeployment, publishUnconfigured: config.PublishUnconfigured, setupGate: make(chan struct{}, 1), mutationGate: make(chan struct{}, 1), ctx: ctx, cancel: stop, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)}, nil } func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway.Registry) (RuntimeProvider, error) { @@ -98,18 +85,15 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. return RuntimeProvider{}, err } copied := *config - if copied.Mode == "" && copied.ProviderKind != "" { - copied.Mode = "nodes" - } - if copied.Mode != "" && copied.Mode != "nodes" && copied.Mode != "direct" { + if copied.ProviderKind == "" || (copied.Mode != "nodes" && copied.Mode != "direct") { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == "direct" && (copied.ProviderKind == "" || copied.LocalNodeID != "" || copied.Suspension != nil) { + if copied.Mode == "direct" && copied.Suspension != nil { return RuntimeProvider{}, sandbox.ErrInvalid } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second || policy.MaxActive < 1 || policy.MaxRetained < policy.MaxActive { + if !sandbox.SupportsCheckpoint(config.Provider) || policy.IdleTimeout < time.Second || policy.Retention < time.Second { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -187,25 +171,6 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p return deployment.Allocation{}, sandbox.ErrInvalid } key := deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment} - if _, err := r.reader.EnvironmentAllocation(ctx, key); errors.Is(err, deployment.ErrNotFound) { - if r.config.AdmissionPaused && r.config.Generation == 0 { - return deployment.Allocation{}, ErrExecutionUnavailable - } - if err := r.computeFreshCapacity(ctx, providerKey); err != nil { - return deployment.Allocation{}, err - } - if policy := r.config.Suspension; policy != nil && r.config.ProviderKind == "" { - count, err := r.reader.CountRetainedAllocations(ctx, providerKey) - if err != nil { - return deployment.Allocation{}, err - } - if count >= int64(policy.MaxRetained) { - return deployment.Allocation{}, ErrExecutionUnavailable - } - } - } else if err != nil { - return deployment.Allocation{}, err - } secret := make([]byte, 32) if _, err := rand.Read(secret); err != nil { return deployment.Allocation{}, err @@ -435,11 +400,3 @@ func runtimeReference(owner deployment.Allocation) sandbox.Reference { func (w *Worker) runManagedRuntimes(ctx context.Context) error { return w.runtimes.run(ctx) } - -// Manager deployments reserve capacity with Session placement before provisioning. -func (r *runtimeLifecycle) computeFreshCapacity(ctx context.Context, key string) error { - if r.config.ProviderKind != "" { - return nil - } - return r.computeCapacity(ctx, key) -} diff --git a/services/core/internal/execution/runtime_manager.go b/services/core/internal/execution/runtime_manager.go index 24f0ec8cb..d7d1af2b2 100644 --- a/services/core/internal/execution/runtime_manager.go +++ b/services/core/internal/execution/runtime_manager.go @@ -79,7 +79,7 @@ func (m *runtimeManager) node(id string) (*runtimeNode, error) { m.mu.Unlock() return nil, errRuntimeTransition } - if m.closed || (m.loadDeployment != nil && m.config.Provider == nil) || (id == "") != (m.config.ProviderKind == "" || m.config.Mode == "direct") { + if m.closed || m.config.Provider == nil || (id == "") != (m.config.Mode == "direct") { m.mu.Unlock() return nil, ErrExecutionUnavailable } @@ -241,10 +241,8 @@ func (m *runtimeManager) run(ctx context.Context) error { } m.running = true m.mu.Unlock() - if m.loadDeployment != nil { - if err := m.deployment.CollectGenerations(ctx); err != nil { - return err - } + if err := m.deployment.CollectGenerations(ctx); err != nil { + return err } if err := m.resetStep(ctx); err != nil { return err @@ -263,10 +261,8 @@ func (m *runtimeManager) run(ctx context.Context) error { case err := <-m.failed: return err case <-ticker.C: - if m.loadDeployment != nil { - if err := m.deployment.CollectGenerations(ctx); err != nil { - return err - } + if err := m.deployment.CollectGenerations(ctx); err != nil { + return err } if err := m.resetStep(ctx); err != nil { return err diff --git a/services/core/internal/execution/runtime_manager_test.go b/services/core/internal/execution/runtime_manager_test.go index bce4dc075..a01528e43 100644 --- a/services/core/internal/execution/runtime_manager_test.go +++ b/services/core/internal/execution/runtime_manager_test.go @@ -19,10 +19,11 @@ func (heldLease) CancelOperations(_ context.Context, cancel context.CancelFunc) } func (heldLease) Close(context.Context) error { return nil } +// testRuntimeManager models an already loaded node deployment. func testRuntimeManager(t *testing.T) *runtimeManager { t.Helper() ctx, cancel := context.WithCancel(t.Context()) - m := &runtimeManager{lease: heldLease{}, config: RuntimeProvider{ProviderKind: "docker"}, ctx: ctx, cancel: cancel, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)} + m := &runtimeManager{lease: heldLease{}, config: RuntimeProvider{ProviderKind: "docker", Mode: "nodes", Provider: &drainFixtureProvider{}}, loadDeployment: func(context.Context) (*RuntimeProvider, error) { return nil, nil }, ctx: ctx, cancel: cancel, nodes: make(map[string]*runtimeNode), failed: make(chan error, 1), inventory: make(chan struct{}, 1)} t.Cleanup(func() { m.stop(); m.drain() }) return m } diff --git a/services/core/internal/execution/runtime_pending.go b/services/core/internal/execution/runtime_pending.go index 254b33877..cee0c94a3 100644 --- a/services/core/internal/execution/runtime_pending.go +++ b/services/core/internal/execution/runtime_pending.go @@ -10,9 +10,6 @@ import ( // provisionPending shares the existing lifecycle owner and serial gate. This // also recovers idle Session creation interrupted after its database commit. func (r *runtimeLifecycle) provisionPending(ctx context.Context) error { - if r.config.AdmissionPaused && r.config.Generation == 0 { - return nil - } rows, err := r.reader.UnallocatedEnvironments(ctx, r.nodeID, r.pendingCursor) if err != nil { return err diff --git a/services/core/internal/execution/runtime_retirement_failure_test.go b/services/core/internal/execution/runtime_retirement_failure_test.go index f5e8e69b0..ed2b63489 100644 --- a/services/core/internal/execution/runtime_retirement_failure_test.go +++ b/services/core/internal/execution/runtime_retirement_failure_test.go @@ -32,11 +32,7 @@ func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { owner, _, _, pool := delayedReadWriter(t, &armed, reading, releaseRead) m := testRuntimeManager(t) m.lease = owner.Lease - m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } m.mutationGate = make(chan struct{}, 1) - // This fixture models an already loaded node deployment; its provider is - // needed only for node admission, not for external sandbox operations. - m.config.Provider = &drainFixtureProvider{} original, err := m.node("retiring") if err != nil { t.Fatal(err) @@ -160,7 +156,8 @@ func TestFailedInventoryRetirementClosesAdmissionAndRetainsGate(t *testing.T) { } } -// No provider method is called by the retirement ownership fixture. +// No provider method is called by the runtime manager fixtures; the provider is +// needed only for node admission. type drainFixtureProvider struct{ sandbox.SandboxProvider } // Done is evaluated only after lockMutation's initial admission check, letting @@ -178,7 +175,6 @@ func (c *mutationWaitContext) Done() <-chan struct{} { func TestFailedManagerRejectsAlreadyWaitingMutation(t *testing.T) { m := testRuntimeManager(t) - m.loadDeployment = func(context.Context) (*RuntimeProvider, error) { return nil, nil } m.mutationGate = make(chan struct{}, 1) m.mutationGate <- struct{}{} ctx, cancel := context.WithCancel(t.Context()) diff --git a/services/core/internal/execution/sandbox_deployment_drain_test.go b/services/core/internal/execution/sandbox_deployment_drain_test.go index 3104220af..d80fbefa0 100644 --- a/services/core/internal/execution/sandbox_deployment_drain_test.go +++ b/services/core/internal/execution/sandbox_deployment_drain_test.go @@ -93,7 +93,7 @@ func testLifecycleCancellationPreservesLease(t *testing.T, mode string) { defer hub.Close() id := uuid.NewString() configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -231,7 +231,7 @@ func TestSandboxDeploymentDrainFailureCannotReactivate(t *testing.T) { defer hub.Close() id := uuid.NewString() configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil })) + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return configuration, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_deployment_setup.go b/services/core/internal/execution/sandbox_deployment_setup.go index 7bff16c58..165198028 100644 --- a/services/core/internal/execution/sandbox_deployment_setup.go +++ b/services/core/internal/execution/sandbox_deployment_setup.go @@ -22,14 +22,11 @@ type PreparedRuntimeDeployment struct { type RuntimeDeploymentPreparer func(context.Context, deployment.Setup) (PreparedRuntimeDeployment, error) // NewDeferredRuntimeProvider enables Web setup for one fixed installation. The -// loader returns nil until selection, then the committed immutable generation. -// Replacement is serialized by the deployment mutation gate and drain flow. -func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error), prepare ...RuntimeDeploymentPreparer) *RuntimeProvider { - config := &RuntimeProvider{InstallationID: installationID, loadDeployment: load} - if len(prepare) == 1 { - config.prepareDeployment = prepare[0] - } - return config +// loader returns nil until selection, then the committed immutable generation; +// prepare validates each new selection before it is stored. Replacement is +// serialized by the deployment mutation gate and drain flow. +func NewDeferredRuntimeProvider(installationID string, load func(context.Context) (*RuntimeProvider, error), prepare RuntimeDeploymentPreparer) *RuntimeProvider { + return &RuntimeProvider{InstallationID: installationID, loadDeployment: load, prepareDeployment: prepare} } func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input sandbox.Selection) (deployment.View, error) { @@ -67,9 +64,6 @@ func (w *Worker) InitializeSandboxDeployment(ctx context.Context, input sandbox. // ensureDeployment serializes the first configuration read without holding the // node map lock across database access. All node workers copy this same snapshot. func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) { - if m.loadDeployment == nil { - return true, nil - } ctx, finish, err := m.enter(parent) if err != nil { return false, err @@ -97,7 +91,7 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) if err != nil || config == nil { return false, err } - if config.InstallationID != m.setupInstallationID || config.LocalNodeID != "" || config.loadDeployment != nil || config.ProviderKind == "" { + if config.InstallationID != m.setupInstallationID || config.loadDeployment != nil { return false, sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) @@ -119,9 +113,6 @@ func (m *runtimeManager) ensureDeployment(parent context.Context) (bool, error) // Preparation is outside the manager mutex and all database transactions. A // rejected candidate cannot retire the current generation or its node lanes. func (m *runtimeManager) prepareCandidate(ctx context.Context, input sandbox.Selection) (PreparedRuntimeDeployment, error) { - if m.prepareDeployment == nil { - return PreparedRuntimeDeployment{}, ErrExecutionUnavailable - } setup, err := m.deploymentService.SetupForSelection(m.setupInstallationID, input) if err != nil { return PreparedRuntimeDeployment{}, err @@ -131,7 +122,7 @@ func (m *runtimeManager) prepareCandidate(ctx context.Context, input sandbox.Sel return PreparedRuntimeDeployment{}, err } config := candidate.Config - if config == nil || config.InstallationID != setup.InstallationID || config.ProviderKind != setup.Provider || config.Mode != setup.Mode || config.CoreURL == "" || config.BackendFingerprint != setup.BackendFingerprint || config.LocalNodeID != "" || config.loadDeployment != nil || config.prepareDeployment != nil { + if config == nil || config.InstallationID != setup.InstallationID || config.ProviderKind != setup.Provider || config.Mode != setup.Mode || config.CoreURL == "" || config.BackendFingerprint != setup.BackendFingerprint || config.loadDeployment != nil || config.prepareDeployment != nil { return PreparedRuntimeDeployment{}, sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) @@ -164,7 +155,7 @@ func (m *runtimeManager) publishDeployment(candidate PreparedRuntimeDeployment, m.mu.Lock() defer m.mu.Unlock() config := *candidate.Config - config.Generation, config.AdmissionPaused = committed.Generation, committed.Reset != nil + config.Generation = committed.Generation if m.switching { m.nodes = make(map[string]*runtimeNode) } diff --git a/services/core/internal/execution/sandbox_deployment_setup_test.go b/services/core/internal/execution/sandbox_deployment_setup_test.go index 35271c554..549f233d3 100644 --- a/services/core/internal/execution/sandbox_deployment_setup_test.go +++ b/services/core/internal/execution/sandbox_deployment_setup_test.go @@ -24,14 +24,14 @@ func TestDeferredSandboxDeploymentLoadsOnceBeforeNodeCreation(t *testing.T) { id := uuid.NewString() var selected atomic.Bool var loads atomic.Int32 - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { loads.Add(1) if !selected.Load() { return nil, nil } return configuration, nil - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -74,7 +74,7 @@ func TestDeferredSandboxDeploymentShutdownCancelsLoad(t *testing.T) { close(entered) <-ctx.Done() return nil, ctx.Err() - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -94,13 +94,13 @@ func TestDeferredSandboxProviderFailureKeepsRecoveryAvailable(t *testing.T) { id := uuid.NewString() available := false loadErr := ErrExecutionUnavailable - configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} + configuration := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { if !available { return nil, loadErr } return configuration, nil - })) + }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -192,7 +192,7 @@ func TestCommittedSandboxCandidatePublishesAfterShutdown(t *testing.T) { hub := node.NewHub(node.HubOptions{}) defer hub.Close() id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -205,7 +205,7 @@ func TestCommittedSandboxCandidatePublishesAfterShutdown(t *testing.T) { m.stop() m.publishDeployment(candidate, deployment.View{InstallationID: id, Generation: 2, Mode: "direct", Provider: "e2b", Reset: &deployment.Reset{}}) m.drain() - if m.config.Generation != 2 || !m.config.AdmissionPaused || published == nil || published.Generation != 2 || !published.AdmissionPaused || m.switching { + if m.config.Generation != 2 || published == nil || published.Generation != 2 || m.switching { t.Fatal("committed candidate was lost during shutdown") } if _, _, err := m.enter(t.Context()); !errors.Is(err, ErrExecutionUnavailable) { diff --git a/services/core/internal/execution/sandbox_deployment_switch.go b/services/core/internal/execution/sandbox_deployment_switch.go index 6e18883cf..0dddd2923 100644 --- a/services/core/internal/execution/sandbox_deployment_switch.go +++ b/services/core/internal/execution/sandbox_deployment_switch.go @@ -9,7 +9,7 @@ import ( ) func (m *runtimeManager) lockMutation(ctx context.Context) (func(), error) { - if m == nil || m.loadDeployment == nil { + if m == nil { return nil, deployment.ErrConflict } m.mu.Lock() @@ -115,7 +115,7 @@ func (m *runtimeManager) activateDeployment(ctx context.Context, expected deploy m.publishEmptyDeployment(expected.InstallationID, expected.Generation) return nil } - if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil || config.LocalNodeID != "" { + if config == nil || config.InstallationID != expected.InstallationID || config.Generation != expected.Generation || config.Mode != expected.Mode || config.ProviderKind != expected.Provider || config.loadDeployment != nil { return sandbox.ErrInvalid } copied, err := validatedRuntimeProvider(config, m.registry) diff --git a/services/core/internal/execution/sandbox_deployment_switch_test.go b/services/core/internal/execution/sandbox_deployment_switch_test.go index 2d31d3d32..551d1ec78 100644 --- a/services/core/internal/execution/sandbox_deployment_switch_test.go +++ b/services/core/internal/execution/sandbox_deployment_switch_test.go @@ -19,7 +19,7 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { defer hub.Close() id := uuid.NewString() config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestSandboxManagerSwitchDrainsBeforeDirectActivation(t *testing.T) { func TestSandboxManagerFailedActivationStaysPaused(t *testing.T) { id := uuid.NewString() - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, errors.New("provider unavailable") }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestSandboxManagerCancelledSwitchCannotResumeBeforeDrain(t *testing.T) { defer hub.Close() id := uuid.NewString() config := &RuntimeProvider{InstallationID: id, ProviderKind: "docker", Mode: "nodes", Generation: 1, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_provider_contract_test.go b/services/core/internal/execution/sandbox_provider_contract_test.go index 531b3fbbc..83d0cdef2 100644 --- a/services/core/internal/execution/sandbox_provider_contract_test.go +++ b/services/core/internal/execution/sandbox_provider_contract_test.go @@ -18,7 +18,7 @@ func TestSandboxProviderRegistrationDoesNotRequireAnExecutionVendorBranch(t *tes id := uuid.NewString() config := &RuntimeProvider{InstallationID: id, ProviderKind: "contract-fixture", Mode: mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: strings.Repeat("a", 64), Provider: &lifecycleOnlySandbox{}} - m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil })) + m, err := newRuntimeManager(Owner{Lease: heldLease{}}, nil, nil, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return config, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_reset.go b/services/core/internal/execution/sandbox_reset.go index f841d4ee8..241d452b0 100644 --- a/services/core/internal/execution/sandbox_reset.go +++ b/services/core/internal/execution/sandbox_reset.go @@ -64,9 +64,6 @@ func (m *runtimeManager) committedView(ctx context.Context) (deployment.View, er // enter m.active, hold a Session/deployment transaction, or call a provider while // waiting for a deployment drain. Each page has both a row and time bound. func (m *runtimeManager) resetStep(parent context.Context) error { - if m.loadDeployment == nil { - return nil - } ctx, cancel := context.WithTimeout(parent, 5*time.Second) defer cancel() return m.resetPage(parent, ctx) diff --git a/services/core/internal/execution/sandbox_reset_test.go b/services/core/internal/execution/sandbox_reset_test.go index 9ec0bb050..16bd12cc9 100644 --- a/services/core/internal/execution/sandbox_reset_test.go +++ b/services/core/internal/execution/sandbox_reset_test.go @@ -101,7 +101,7 @@ func TestSandboxResetPageTimeoutRecoversCommittedOwner(t *testing.T) { return nil, err } return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + }, unusedPreparation(t)) m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), config) if err != nil { t.Fatal(err) @@ -215,7 +215,7 @@ func TestSandboxResetPublishesCommittedGenerationWithoutReading(t *testing.T) { return nil, err } return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + }, unusedPreparation(t)) var published []uint64 config.PublishUnconfigured = func(generation uint64) { if committedReads != 0 { @@ -255,10 +255,10 @@ func TestSandboxResetPublishesCommittedGenerationWithoutReading(t *testing.T) { func TestCommittedResetViewStopsOwnerWithoutLease(t *testing.T) { id := uuid.NewString() reader := &strictDeploymentReader{t: t, snapshot: func(context.Context) (deployment.Snapshot, error) { - return deployment.Snapshot{Record: deployment.Record{InstallationID: id, WebManaged: true, Generation: 1}}, nil + return deployment.Snapshot{Record: deployment.Record{InstallationID: id, Generation: 1}}, nil }} deployments, operations := deploymentOperations(t, &strictDeploymentStorage{t: t}, reader, &strictExecutionStorage{t: t}) - m, err := newRuntimeManager(Owner{Lease: lostLease{}, Deployment: operations}, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(Owner{Lease: lostLease{}, Deployment: operations}, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } @@ -280,7 +280,7 @@ func TestCommittedResetViewStopsOwnerWithoutLease(t *testing.T) { func TestSandboxResetChangesReturnViewReadAfterCommit(t *testing.T) { owner, deployments, reader := resetManager(t) id := initializeE2BDeployment(t, owner) - m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil })) + m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), NewDeferredRuntimeProvider(id, func(context.Context) (*RuntimeProvider, error) { return nil, nil }, unusedPreparation(t))) if err != nil { t.Fatal(err) } diff --git a/services/core/internal/execution/sandbox_snapshot_budget_test.go b/services/core/internal/execution/sandbox_snapshot_budget_test.go index 467596e9a..60a0e1067 100644 --- a/services/core/internal/execution/sandbox_snapshot_budget_test.go +++ b/services/core/internal/execution/sandbox_snapshot_budget_test.go @@ -73,7 +73,7 @@ func TestSandboxResetSnapshotFitsPageBudget(t *testing.T) { return nil, err } return &RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: hub.Proxy(uuid.NewString(), "docker", docker.Operations(), 1)}, nil - }) + }, unusedPreparation(t)) m, err := newRuntimeManager(owner, deployments, reader, nil, runtimegateway.NewRegistry(), configuration) if err != nil { t.Fatal(err) diff --git a/services/core/internal/execution/worker.go b/services/core/internal/execution/worker.go index 137dd1a82..839009390 100644 --- a/services/core/internal/execution/worker.go +++ b/services/core/internal/execution/worker.go @@ -9,7 +9,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -85,19 +84,12 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher, owner Owner) (_ *W worker.runtimes.stop() } }() - } - var process *deployment.ProcessDeployment - if worker.runtimes != nil && worker.runtimes.loadDeployment == nil { - config := worker.runtimes.config - process = &deployment.ProcessDeployment{ProviderKind: config.ProviderKind, LocalNodeID: config.LocalNodeID, LocalCredentialSHA256: config.LocalCredentialSHA256, LocalMaxActive: config.LocalMaxActive, LocalMaxRetained: config.LocalMaxRetained, InstallationID: config.InstallationID, BackendFingerprint: config.BackendFingerprint, AdmissionPaused: config.AdmissionPaused} - } - if worker.runtimes != nil && worker.runtimes.loadDeployment != nil { err = owner.Deployment.Claim(ctx, worker.runtimes.setupInstallationID) if err == nil { _, err = worker.runtimes.ensureDeployment(ctx) } } else { - err = owner.Deployment.ConfigureProcess(ctx, process) + err = owner.Deployment.RequireUnclaimed(ctx) } if err != nil { return nil, err diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 08e2569cb..a6ae895e1 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -540,19 +540,3 @@ func (s *Store) Activity(ctx context.Context, allocationID string) (deployment.A } return loadActivity(ctx, s.pool.Queries(), id) } - -func (s *Store) CountComputeReservations(ctx context.Context, installationID string) (int64, error) { - id, err := parseID(installationID) - if err != nil { - return 0, err - } - return s.pool.Queries().CountRuntimeComputeReservations(ctx, id) -} - -func (s *Store) CountRetainedAllocations(ctx context.Context, installationID string) (int64, error) { - id, err := parseID(installationID) - if err != nil { - return 0, err - } - return s.pool.Queries().CountRuntimeRetainedAllocations(ctx, id) -} diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go index b310e6688..84eed9174 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/records.go +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -59,8 +59,8 @@ func translate(err error) error { // credentialcrypto.ErrUnavailable; a credential the key cannot open or // authenticate is an internal decryption error. func record(d sqlc.RuntimeDeployment, cipher *credentialcrypto.Cipher, open bool) deployment.Record { - r := deployment.Record{InstallationID: uuidString(d.InstallationID), WebManaged: d.WebManaged, Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, - Generation: uint64(d.Generation), OwnerEpoch: uint64(d.OwnerEpoch), Mode: d.Mode, AdmissionPaused: d.AdmissionPaused, LocalNodeID: uuidString(d.LocalNodeID), + r := deployment.Record{InstallationID: uuidString(d.InstallationID), Provider: d.ProviderKind, BackendFingerprint: d.BackendFingerprint, + Generation: uint64(d.Generation), OwnerEpoch: uint64(d.OwnerEpoch), Mode: d.Mode, IdleSeconds: d.IdleSeconds, RetentionSeconds: d.RetentionSeconds, Specification: d.Specification, Configuration: sandbox.ConfigurationRecord{Public: d.ProviderConfig, Metadata: d.ProviderMetadata}, CredentialStored: len(d.ProviderCredential) > 0} if r.CredentialStored && open { diff --git a/services/core/internal/persistence/postgres/deploymentpg/reset_test.go b/services/core/internal/persistence/postgres/deploymentpg/reset_test.go index 9c46bb524..5e242387a 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/reset_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/reset_test.go @@ -125,8 +125,8 @@ func TestResetDeadlineAndCancel(t *testing.T) { if err := changes.CancelReset(admin(t), installation, view.Generation); err != nil { t.Fatal(err) } - if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE admission_paused OR reset_clear IS NOT NULL OR reset_audit IS NOT NULL"); paused != 0 { - t.Fatal("cancellation left the reset or paused admission") + if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE reset_clear IS NOT NULL OR reset_audit IS NOT NULL"); paused != 0 { + t.Fatal("cancellation left the reset") } if got, want := resetAudit(t, f), []string{"reset_start fixture-admin", "reset_deadline fixture-admin", "reset_cancel fixture-admin"}; !slices.Equal(got, want) { t.Fatalf("audit = %q, want %q", got, want) @@ -146,7 +146,7 @@ func TestResetWritesNothingWithoutAuditOrLease(t *testing.T) { if err := closed.StartReset(admin(t), installation, request); !errors.Is(err, pgunit.ErrLeaseClosed) { t.Fatalf("StartReset on a closed lease = %v", err) } - if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE admission_paused OR reset_clear IS NOT NULL"); paused != 0 { + if paused := resetCount(t, f, "SELECT count(*) FROM runtime_deployment WHERE reset_clear IS NOT NULL"); paused != 0 { t.Fatal("a rejected reset paused admission") } } diff --git a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go index f9ac218b8..6fee76efc 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/setup_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/setup_test.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "errors" "fmt" - "strings" "sync" "testing" @@ -112,32 +111,20 @@ func TestSandboxDeploymentSetupConcurrentSelection(t *testing.T) { } } -func TestSandboxDeploymentSetupRejectsFileManagedAndUnleasedWrites(t *testing.T) { +func TestSandboxDeploymentSetupRejectsUnleasedWrites(t *testing.T) { f := newFixture(t) - input := sandbox.Selection{DeploymentSpec: testSpecification("docker"), Provider: "docker"} - process := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), ProviderKind: "docker", - LocalNodeID: uuid.NewString(), LocalCredentialSHA256: setupDigest("local-node-credential"), LocalMaxActive: 4, LocalMaxRetained: 16} + id := uuid.NewString() // Deployment changes run only on the execution lease; a closed one writes nothing. closed := setupClosedExecution(t, f) - if err := closed.ConfigureProcess(t.Context(), &process); !errors.Is(err, pgunit.ErrLeaseClosed) { - t.Fatal("unleased process configuration accepted", err) + if err := closed.Claim(t.Context(), id); !errors.Is(err, pgunit.ErrLeaseClosed) { + t.Fatal("unleased claim accepted", err) } - if _, err := closed.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, pgunit.ErrLeaseClosed) { + if _, err := closed.Initialize(t.Context(), id, sandbox.Selection{DeploymentSpec: testSpecification("docker"), Provider: "docker"}); !errors.Is(err, pgunit.ErrLeaseClosed) { t.Fatal("unleased setup accepted", err) } if view, err := f.service.View(t.Context()); err != nil || view.InstallationID != "" || view.Provider != "" { t.Fatal("unleased writes changed the deployment", view, err) } - changes, _ := f.execution(t) - if err := changes.ConfigureProcess(t.Context(), &process); err != nil { - t.Fatal(err) - } - if _, err := changes.Initialize(t.Context(), process.InstallationID, input); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file-managed deployment changed", err) - } - if err := changes.Claim(t.Context(), process.InstallationID); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file-managed deployment adopted", err) - } } func TestSandboxSelectionRejectsWhitespaceInE2BCredential(t *testing.T) { @@ -183,54 +170,3 @@ func TestSandboxE2BEndpointPersistenceAndOnlineSwitch(t *testing.T) { t.Fatal("online endpoint switch failed", changed, err) } } - -func TestRuntimeDeploymentRequiresMaintenanceBeforeIdentityChange(t *testing.T) { - f := newFixture(t) - old := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} - if err := setupClosedExecution(t, f).ConfigureProcess(t.Context(), &old); !errors.Is(err, pgunit.ErrLeaseClosed) { - t.Fatal("unleased configuration accepted", err) - } - changes, _ := f.execution(t) - configure := func(selected *deployment.ProcessDeployment) { - t.Helper() - if err := changes.ConfigureProcess(t.Context(), selected); err != nil { - t.Fatal(err) - } - } - configure(&old) - for _, changeID := range []bool{false, true} { - next := old - if changeID { - next.InstallationID = uuid.NewString() - } else { - next.BackendFingerprint = strings.Repeat("b", 64) - } - next.AdmissionPaused = true - if err := changes.ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "maintenance") { - t.Fatal("identity changed before prior maintenance", err) - } - } - old.AdmissionPaused = true - configure(&old) - next := old - next.BackendFingerprint = strings.Repeat("b", 64) - next.AdmissionPaused = false - if err := changes.ConfigureProcess(t.Context(), &next); err == nil { - t.Fatal("switch reopened creation in same operation") - } - next.AdmissionPaused = true - configure(&next) - var id, fingerprint string - var maintenance bool - if err := f.pool.QueryRow(t.Context(), "SELECT installation_id::text,backend_fingerprint,admission_paused FROM runtime_deployment").Scan(&id, &fingerprint, &maintenance); err != nil || id != next.InstallationID || fingerprint != next.BackendFingerprint || !maintenance { - t.Fatal("switch identity not durable", id, fingerprint, maintenance, err) - } - configure(nil) - // Disabling the configured adapter must not forget the old maintenance state. - next.AdmissionPaused = false - configure(&next) - another := deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64), AdmissionPaused: true} - if err := changes.ConfigureProcess(t.Context(), &another); err == nil { - t.Fatal("nil selection erased the maintenance prerequisite") - } -} diff --git a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go index f739faaec..304b9121c 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/specification_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/specification_test.go @@ -16,7 +16,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/e2b" ) -func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *testing.T) { +func TestSandboxSpecificationRoundTripAndClaimStays(t *testing.T) { for _, provider := range []string{"docker", "microsandbox", "e2b"} { t.Run(provider, func(t *testing.T) { f := newFixture(t) @@ -44,11 +44,8 @@ func TestSandboxSpecificationRoundTripAndFileConfigurationCannotOverride(t *test if _, err := changes.Initialize(t.Context(), view.InstallationID, changed); !errors.Is(err, deployment.ErrConflict) { t.Fatal("initial setup silently resized a configured deployment", err) } - file := deployment.ProcessDeployment{InstallationID: view.InstallationID, BackendFingerprint: setup.BackendFingerprint, ProviderKind: provider, AdmissionPaused: true} - for _, candidate := range []*deployment.ProcessDeployment{nil, &file} { - if err := changes.ConfigureProcess(t.Context(), candidate); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("file configuration replaced database ownership", err) - } + if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("an owner without sandbox runtimes started on a claimed deployment", err) } after, err := f.service.View(t.Context()) if err != nil || !reflect.DeepEqual(after, view) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/tx.go b/services/core/internal/persistence/postgres/deploymentpg/tx.go index 6145dd13b..e0952eb19 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/tx.go +++ b/services/core/internal/persistence/postgres/deploymentpg/tx.go @@ -297,25 +297,6 @@ func (t *deploymentTx) ClaimInstallation(installationID string) error { return t.q.ClaimWebSandboxDeployment(t.ctx, id) } -func (t *deploymentTx) SetProcessDeployment(installationID, backendFingerprint string, admissionPaused bool) error { - id, err := parseID(installationID) - if err != nil { - return err - } - return t.q.SetRuntimeDeployment(t.ctx, sqlc.SetRuntimeDeploymentParams{InstallationID: id, BackendFingerprint: backendFingerprint, AdmissionPaused: admissionPaused}) -} - -func (t *deploymentTx) SetManagerDeployment(provider, localNodeID string) error { - var local pgtype.UUID - if localNodeID != "" { - var err error - if local, err = parseID(localNodeID); err != nil { - return err - } - } - return t.q.SetRuntimeManagerDeployment(t.ctx, sqlc.SetRuntimeManagerDeploymentParams{ProviderKind: provider, LocalNodeID: local}) -} - func (t *deploymentTx) SaveSelection(selection deployment.SelectionRecord) error { if selection.Generation > math.MaxInt64 { return deployment.ErrInvalidInput diff --git a/services/core/internal/persistence/postgres/placementpg/placementpg.go b/services/core/internal/persistence/postgres/placementpg/placementpg.go index f67cbf294..73fc46b35 100644 --- a/services/core/internal/persistence/postgres/placementpg/placementpg.go +++ b/services/core/internal/persistence/postgres/placementpg/placementpg.go @@ -25,8 +25,7 @@ func LockDeployment(ctx context.Context, q *sqlc.Queries) (placement.Deployment, } return placement.Deployment{ InstallationID: uuidString(d.InstallationID), Provider: d.ProviderKind, Mode: d.Mode, - Generation: uint64(d.Generation), WebManaged: d.WebManaged, AdmissionPaused: d.AdmissionPaused, - Resetting: d.ResetClear.Valid, Specification: d.Specification, + Generation: uint64(d.Generation), Resetting: d.ResetClear.Valid, Specification: d.Specification, }, nil } diff --git a/services/core/internal/persistence/postgres/sessionpg/creation_test.go b/services/core/internal/persistence/postgres/sessionpg/creation_test.go index 65ca06140..1552275b8 100644 --- a/services/core/internal/persistence/postgres/sessionpg/creation_test.go +++ b/services/core/internal/persistence/postgres/sessionpg/creation_test.go @@ -473,8 +473,8 @@ func TestEnvironmentCreationReservesItsInitialInput(t *testing.T) { } } -// Hosted creation checks admission on the locked deployment, so it sees -// maintenance committed while it waited, and places after creating the +// Hosted creation checks admission on the locked deployment, so it sees a +// reset committed while it waited, and places after creating the // Environment, rolling both back when no node is available. A retry admits // nothing and still returns its Session. func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { @@ -505,19 +505,20 @@ func TestHostedCreationAdmitsAndPlacesUnderTheDeploymentLock(t *testing.T) { done <- err }() awaitBlocked(ctx, t, pool, holder) - if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, admission_paused=true", uuid.New(), strings.Repeat("a", 64)); err != nil { + if _, err := tx.Exec(ctx, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2, provider_kind='docker', mode='nodes', generation=1, + reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`, uuid.New(), strings.Repeat("a", 64)); err != nil { t.Fatal(err) } if err := tx.Commit(ctx); err != nil { t.Fatal(err) } - if err := <-done; !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("creation bypassed committed maintenance", err) + if err := <-done; !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("creation bypassed the committed reset", err) } if retry, err := service.CreateSession(ctx, tenant, hosted("existing")); err != nil || retry.Created || retry.Session.ID != existing.Session.ID { t.Fatal("retry ran admission", retry, err) } - exec(t, pool, "UPDATE runtime_deployment SET admission_paused=false, web_managed=true") + exec(t, pool, "UPDATE runtime_deployment SET reset_clear=NULL, reset_requested_at=NULL, reset_forced_at=NULL, reset_audit=NULL, provider_kind='', mode=''") if _, err := service.CreateSession(ctx, tenant, hosted("unplaced")); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("placement without a node", err) } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index 5d60ab7fc..ab92077b5 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -75,15 +75,6 @@ func (r *Registry) Lookup(kind string) (Adapter, error) { return a, nil } -// IsNode reports whether the provider runs on enrolled sandbox nodes. -func (r *Registry) IsNode(kind string) (bool, error) { - a, err := r.Lookup(kind) - if err != nil { - return false, err - } - return a.Mode == "nodes", nil -} - // SupportsCheckpoint reports whether the provider declares checkpoint suspension. func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { a, err := r.Lookup(kind) diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 97c1539df..6b3078902 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -28,9 +28,8 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { } a, err := registry.Lookup(tc.kind) checkpoint, checkpointErr := registry.SupportsCheckpoint(tc.kind) - isNode, nodeErr := registry.IsNode(tc.kind) - if err != nil || checkpointErr != nil || nodeErr != nil || checkpoint != tc.checkpoint || isNode != (tc.mode == "nodes") || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { - t.Fatal("inconsistent construction/capability registration", err, checkpointErr, nodeErr) + if err != nil || checkpointErr != nil || checkpoint != tc.checkpoint || (a.BuildLocal != nil) != (tc.mode == "nodes") || (a.BuildDirect != nil) != (tc.mode == "direct") { + t.Fatal("inconsistent construction/capability registration", err, checkpointErr) } }) } @@ -71,10 +70,9 @@ func TestNewRegistrationDoesNotNeedCoreDispatchChanges(t *testing.T) { // Registration is test-local: production registrations are fixed, never plugins. registry.adapters[kind] = registry.adapters["docker"] s, err := registry.Normalize(sandbox.Selection{Provider: kind, DeploymentSpec: validRegistrationSpec()}) - isNode, nodeErr := registry.IsNode(kind) checkpoint, checkpointErr := registry.SupportsCheckpoint(kind) - if err != nil || nodeErr != nil || checkpointErr != nil || s.Provider != kind || !isNode || checkpoint { - t.Fatal("new entry did not follow shared boundary", err, nodeErr, checkpointErr) + if err != nil || checkpointErr != nil || s.Provider != kind || checkpoint { + t.Fatal("new entry did not follow shared boundary", err, checkpointErr) } d, err := registry.Describe(kind, uuid.NewString()) if err != nil || d.Mode != "nodes" || d.IdleSeconds != 0 { @@ -113,9 +111,6 @@ func TestCapabilityLookupsReportFailures(t *testing.T) { invalid.Operations = nil registry.adapters["invalid-registration"] = invalid for kind, want := range map[string]error{"unregistered": ErrUnknownProvider, "invalid-registration": providercontract.ErrContract} { - if _, err := registry.IsNode(kind); !errors.Is(err, want) { - t.Fatal(kind, "IsNode", err) - } if _, err := registry.SupportsCheckpoint(kind); !errors.Is(err, want) { t.Fatal(kind, "SupportsCheckpoint", err) } diff --git a/services/core/internal/sessions/creation_test.go b/services/core/internal/sessions/creation_test.go index 865fee9e1..7fe34ff68 100644 --- a/services/core/internal/sessions/creation_test.go +++ b/services/core/internal/sessions/creation_test.go @@ -449,7 +449,7 @@ func TestCreateSession(t *testing.T) { tx := newCreationTx(t, hostedSession, true) tx.lockDeployment = returns(placement.Deployment{InstallationID: "installation", Provider: "docker", Specification: json.RawMessage(`{}`)}) tx.createEnvironment = returns("environment") - tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1}}) + tx.loadNodes = returns([]placement.Node{{ID: "node", Online: true, ServingReady: true, ReadyGeneration: &ready, MaxActive: 1, MaxRetained: 1, CoreURL: rules.PublicURL()}}) tx.reservePlacement = done _, err, calls := runCreation(t, rules, tx, creationInput("openai_hosted")) want := []string{"UpsertSession create", "LockDeployment", "CreateEnvironment", "LoadNodes", "ReservePlacement node 5", "AuditCreation session:session environment:environment:session", "LoadSession"} diff --git a/services/core/migrations/000093_web_deployment_only.sql b/services/core/migrations/000093_web_deployment_only.sql new file mode 100644 index 000000000..54886e099 --- /dev/null +++ b/services/core/migrations/000093_web_deployment_only.sql @@ -0,0 +1,78 @@ +-- +goose Up +-- Web setup is the only writer of the installation, so a recorded installation +-- is Web-managed, no deployment names a process-configured local node, every +-- node has the Core address it enrolled with, and only a sandbox reset pauses +-- admission. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM runtime_deployment WHERE local_node_id IS NOT NULL OR (installation_id IS NOT NULL AND NOT web_managed) + OR admission_paused <> (reset_clear IS NOT NULL)) + OR EXISTS (SELECT 1 FROM runtime_nodes WHERE removed_at IS NULL AND core_url = '') THEN + RAISE EXCEPTION 'Cannot upgrade: the sandbox deployment was configured outside Web setup, or a live node has no Core address. Reset the sandbox in Web, then upgrade'; + END IF; +END $$; +-- +goose StatementEnd +ALTER TABLE runtime_deployment + DROP CONSTRAINT runtime_deployment_identity_check, + DROP CONSTRAINT runtime_deployment_setup_check, + DROP CONSTRAINT runtime_deployment_reset_check, + DROP CONSTRAINT runtime_deployment_reset_admission_check, + DROP COLUMN web_managed, + DROP COLUMN local_node_id, + DROP COLUMN admission_paused, + ADD CONSTRAINT runtime_deployment_identity_check CHECK ( + (installation_id IS NULL AND backend_fingerprint = '') OR + (installation_id IS NOT NULL AND backend_fingerprint ~ '^[0-9a-f]{64}$') OR + (installation_id IS NOT NULL AND provider_kind = '' AND backend_fingerprint = '') + ), + ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + installation_id IS NULL OR + (provider_kind = '' AND mode = '' AND generation >= 0 AND idle_seconds = 0 AND retention_seconds = 0) OR + (provider_kind <> '' AND mode IN ('nodes','direct') AND generation > 0 AND + ((idle_seconds = 0 AND retention_seconds = 0) OR (idle_seconds > 0 AND retention_seconds > 0))) + ), + ADD CONSTRAINT runtime_deployment_reset_check CHECK ( + (reset_clear IS NULL AND reset_requested_at IS NULL AND reset_deadline_at IS NULL + AND reset_forced_at IS NULL AND reset_audit IS NULL) + OR (reset_clear IS NOT NULL AND installation_id IS NOT NULL AND provider_kind <> '' + AND reset_requested_at IS NOT NULL AND reset_audit IS NOT NULL + AND jsonb_typeof(reset_audit) = 'object' + AND ((reset_clear = 'auto' AND reset_deadline_at IS NOT NULL AND reset_forced_at IS NULL) + OR (reset_clear = 'force' AND reset_forced_at IS NOT NULL))) + ); + +-- +goose Down +ALTER TABLE runtime_deployment + DROP CONSTRAINT runtime_deployment_identity_check, + DROP CONSTRAINT runtime_deployment_setup_check, + DROP CONSTRAINT runtime_deployment_reset_check, + ADD COLUMN web_managed boolean NOT NULL DEFAULT false, + ADD COLUMN local_node_id uuid, + ADD COLUMN admission_paused boolean NOT NULL DEFAULT false; +UPDATE runtime_deployment SET web_managed = installation_id IS NOT NULL, admission_paused = reset_clear IS NOT NULL; +ALTER TABLE runtime_deployment + ADD CONSTRAINT runtime_deployment_identity_check CHECK ( + (installation_id IS NULL AND backend_fingerprint = '') OR + (installation_id IS NOT NULL AND backend_fingerprint ~ '^[0-9a-f]{64}$') OR + (web_managed AND installation_id IS NOT NULL AND provider_kind = '' AND backend_fingerprint = '') + ), + ADD CONSTRAINT runtime_deployment_setup_check CHECK ( + NOT web_managed OR (local_node_id IS NULL AND ( + (provider_kind = '' AND mode = '' AND generation >= 0 AND idle_seconds = 0 AND retention_seconds = 0) OR + (provider_kind <> '' AND mode IN ('nodes','direct') AND generation > 0 AND + ((idle_seconds = 0 AND retention_seconds = 0) OR (idle_seconds > 0 AND retention_seconds > 0))) + )) + ), + ADD CONSTRAINT runtime_deployment_reset_check CHECK ( + (reset_clear IS NULL AND reset_requested_at IS NULL AND reset_deadline_at IS NULL + AND reset_forced_at IS NULL AND reset_audit IS NULL) + OR (reset_clear IS NOT NULL AND web_managed AND provider_kind <> '' + AND reset_requested_at IS NOT NULL AND reset_audit IS NOT NULL + AND jsonb_typeof(reset_audit) = 'object' + AND ((reset_clear = 'auto' AND reset_deadline_at IS NOT NULL AND reset_forced_at IS NULL) + OR (reset_clear = 'force' AND reset_forced_at IS NOT NULL))) + ), + ADD CONSTRAINT runtime_deployment_reset_admission_check CHECK ( + NOT web_managed OR admission_paused = (reset_clear IS NOT NULL) + ); diff --git a/services/core/tests/integration/admin_session_archive_race_test.go b/services/core/tests/integration/admin_session_archive_race_test.go index e973a7c72..177dc8ecb 100644 --- a/services/core/tests/integration/admin_session_archive_race_test.go +++ b/services/core/tests/integration/admin_session_archive_race_test.go @@ -82,11 +82,3 @@ func TestManagedSessionArchiveOrdersConcurrentInput(t *testing.T) { } } } - -func TestManagedSessionArchiveRejectsFileManagedDeployment(t *testing.T) { - s, w, _ := managerFixture(t, 1, 1) - tenant, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - if _, err := deploymentExecution(t, w).ArchiveSession(adminDeleteContext(t.Context(), tenant, uuid.NewString()), tenant, session.ID, 0); !errors.Is(err, deployment.ErrConflict) { - t.Fatal("archive accepted file-managed deployment", err) - } -} diff --git a/services/core/tests/integration/admin_session_archive_worker_http_test.go b/services/core/tests/integration/admin_session_archive_worker_http_test.go index 00f830760..8e42cf4e7 100644 --- a/services/core/tests/integration/admin_session_archive_worker_http_test.go +++ b/services/core/tests/integration/admin_session_archive_worker_http_test.go @@ -41,7 +41,7 @@ func TestAdminSessionArchiveWorkerHTTPPostgres(t *testing.T) { provider := &lifecycleProvider{resources: map[string]sandbox.Info{}} deployments := deploymentService(t, s) providerConfig := func(setup deployment.Setup) *execution.RuntimeProvider { - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} } configuration := execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { setup, err := deployments.Setup(ctx) diff --git a/services/core/tests/integration/credential_matrix_http_test.go b/services/core/tests/integration/credential_matrix_http_test.go index d424fc7d4..068423282 100644 --- a/services/core/tests/integration/credential_matrix_http_test.go +++ b/services/core/tests/integration/credential_matrix_http_test.go @@ -95,9 +95,9 @@ func TestCredentialNamespaceMatrix(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: provider}, nil }, func(_ context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider}}, nil }) worker := startWorker(t, ctx, s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) var stop sync.Once diff --git a/services/core/tests/integration/device_bootstrap_binding_test.go b/services/core/tests/integration/device_bootstrap_binding_test.go index 597001774..bf28d963d 100644 --- a/services/core/tests/integration/device_bootstrap_binding_test.go +++ b/services/core/tests/integration/device_bootstrap_binding_test.go @@ -25,7 +25,7 @@ func TestDeviceCredentialCarriesPersistedAllocationNode(t *testing.T) { t.Fatal(err) } onlineManagerNode(t, s, remote) - for _, nodeID := range []string{d.LocalNodeID, remote} { + for _, nodeID := range []string{d.NodeID, remote} { t.Run(nodeID, func(t *testing.T) { tenant, bearer := uuid.NewString(), uuid.NewString() session, err := createSessionOnNode(t, s, tenant, managerSessionInput(uuid.NewString()), nodeID) diff --git a/services/core/tests/integration/hosted_initialization_failure_public_test.go b/services/core/tests/integration/hosted_initialization_failure_public_test.go index 543f11500..4364a0e8b 100644 --- a/services/core/tests/integration/hosted_initialization_failure_public_test.go +++ b/services/core/tests/integration/hosted_initialization_failure_public_test.go @@ -98,14 +98,17 @@ func (p *hostedFailureProvider) prepare(request proto.RuntimePreparePayload, _ [ return completedInitialization(request, nil) } -func hostedFailureStore(t *testing.T) *Store { +// hostedFailureStore returns a store whose Web deployment is claimed by the +// returned installation. +func hostedFailureStore(t *testing.T) (*Store, string) { t.Helper() _, pool := newManagedTestStore(t) cipher, err := credentialcrypto.New(bytes.Repeat([]byte{7}, 32)) if err != nil { t.Fatal(err) } - return NewWithCredentialCipher(pool, cipher) + s := NewWithCredentialCipher(pool, cipher) + return s, webDeployment(t, s, "e2b") } func hostedFailureSession(t *testing.T, s *Store, tenant string, input sessions.CreateSession) (sessions.Session, sessions.Environment) { @@ -126,10 +129,9 @@ func hostedFailureSession(t *testing.T, s *Store, tenant string, input sessions. return session, environment } -func failHostedInitialization(t *testing.T, s *Store, tenant string, environment sessions.Environment, p *hostedFailureProvider) { +func failHostedInitialization(t *testing.T, s *Store, key, tenant string, environment sessions.Environment, p *hostedFailureProvider) { t.Helper() - key := uuid.NewString() - w, _ := managedWorkerMode(t, s, key, p, false, true) + w, _ := managedWorkerMode(t, s, key, p, true) if _, err := w.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { t.Fatal(err) } @@ -180,12 +182,12 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { "Failed to provision environment: Skill installation failed", []string{"configure", "skill"}}, } { t.Run(test.name, func(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, test.input) p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: test.p.fail, skip: test.p.skip, result: test.p.result, err: test.p.err} - failHostedInitialization(t, s, tenant, environment, p) + failHostedInitialization(t, s, key, tenant, environment, p) if !reflect.DeepEqual(p.steps, test.steps) || p.kills != 0 || p.commandCalls.Load() != 0 { t.Fatal("failed initialization continued or reclaimed compute", p.steps, p.kills) } @@ -245,7 +247,7 @@ func TestHostedInitializationFailureRecordsSafeSessionFailure(t *testing.T) { // A pending initial input settles exactly as before; the one failed snapshot // carries both that settlement and the provisioning failure. func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant := uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "exit 3"}}}, @@ -253,7 +255,7 @@ func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { }) p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", result: failedInitialization(3)} - failHostedInitialization(t, s, tenant, environment, p) + failHostedInitialization(t, s, key, tenant, environment, p) read, err := sessionAdapter(s).GetSession(t.Context(), tenant, session.ID) if err != nil || read.PendingInput || read.EnvironmentInputActivity == nil || read.EnvironmentInputActivity.Status != "failed" || read.EnvironmentInputActivity.Failure != "environment_unavailable" || read.EnvironmentFailure == nil { @@ -279,20 +281,19 @@ func TestHostedInitializationFailureSettlesPendingInitialInput(t *testing.T) { // stream ends after agent.session.failed; later input gets the observed 409; // delete succeeds; tenant B sees nothing; the canary never appears. func TestHostedInitializationFailurePublicHTTP(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) tenant, token, foreign := uuid.NewString(), uuid.NewString(), uuid.NewString() session, environment := hostedFailureSession(t, s, tenant, sessions.CreateSession{ Initialization: environmentconfig.Setup{Commands: []environmentconfig.SetupCommand{{Command: "echo " + hostedFailureCanary + "; exit 3"}}}, Metadata: map[string]string{"case": "setup-exit3"}, }) - key := uuid.NewString() // A failed typed Runtime receipt exposes only a safe status. p := &hostedFailureProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, fail: "setup", result: failedInitialization(3)} logs := &lockedBuffer{} previous := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) t.Cleanup(func() { slog.SetDefault(previous) }) - w, _ := managedWorkerMode(t, s, key, p, false, true) + w, _ := managedWorkerMode(t, s, key, p, true) auth := newTestAuthenticator(t, []testAPIKey{ {OrganizationID: "test-org", ProjectID: tenant, SubjectKind: "service_account", SubjectID: "test-runner", TokenSHA256: runtimedevice.HashCredential(token), TenantID: tenant}, {OrganizationID: "test-org", ProjectID: uuid.NewString(), SubjectKind: "service_account", SubjectID: "tenant-b", TokenSHA256: runtimedevice.HashCredential(foreign), TenantID: uuid.NewString()}, diff --git a/services/core/tests/integration/runtime_adoption_test.go b/services/core/tests/integration/runtime_adoption_test.go deleted file mode 100644 index 76ae44516..000000000 --- a/services/core/tests/integration/runtime_adoption_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package integration - -import ( - "fmt" - "reflect" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" - "github.com/google/uuid" -) - -func nodelessAllocationFixture(t *testing.T) (*Store, *Store, deployment.ProcessDeployment, deployment.Allocation) { - t.Helper() - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - d := deploymentSelection() - deploymentConfigure(t, w, &d) - tenant := uuid.NewString() - _, e := localEnvironment(t, s, tenant) - a, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: e.ID}, d.InstallationID, runtimedevice.HashCredential("runtime")) - if err != nil { - t.Fatal(err) - } - d.ProviderKind = "docker" - d.LocalNodeID = uuid.NewString() - d.LocalCredentialSHA256 = runtimedevice.HashCredential("node") - d.LocalMaxActive, d.LocalMaxRetained = 4, 16 - return s, w, d, a -} -func requireNoNodeBinding(t *testing.T, s *Store) { - t.Helper() - var nodes, placements, bound int - var kind string - if err := s.pool.QueryRow(t.Context(), `SELECT provider_kind,(SELECT count(*) FROM runtime_nodes),(SELECT count(*) FROM runtime_placements),(SELECT count(*) FROM runtime_allocations WHERE node_id IS NOT NULL) FROM runtime_deployment`).Scan(&kind, &nodes, &placements, &bound); err != nil { - t.Fatal(err) - } - if kind != "" || nodes != 0 || placements != 0 || bound != 0 { - t.Fatal("partial adoption", kind, nodes, placements, bound) - } -} -func TestHistoricalRuntimeResourcesCannotBeAdopted(t *testing.T) { - for _, state := range []string{"creating", "running", "cleanup_pending", "released"} { - for _, pending := range []bool{false, true} { - t.Run(fmt.Sprintf("%s/pending=%t", state, pending), func(t *testing.T) { - s, w, d, a := nodelessAllocationFixture(t) - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET state=$2,create_settled=($2='released'),released_at=CASE WHEN $2='released' THEN clock_timestamp() ELSE NULL END WHERE id=$1`, a.ID, state) - if pending { - _, _ = localEnvironment(t, s, a.TenantID) - } - before, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.EnvironmentID}) - if err != nil { - t.Fatal(err) - } - err = deploymentExecution(t, w).ConfigureProcess(t.Context(), &d) - if state == "released" && !pending { - if err != nil { - t.Fatal("released history blocked fresh configuration", err) - } - } else { - if err == nil { - t.Fatal("historical resources adopted") - } - requireNoNodeBinding(t, s) - } - after, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: a.TenantID, EnvironmentID: a.EnvironmentID}) - if err != nil || !reflect.DeepEqual(before, after) { - t.Fatal("retained receipt changed", err) - } - }) - } - } -} diff --git a/services/core/tests/integration/runtime_capabilities_pending_test.go b/services/core/tests/integration/runtime_capabilities_pending_test.go index 5bbe67cf6..3e3c01d2d 100644 --- a/services/core/tests/integration/runtime_capabilities_pending_test.go +++ b/services/core/tests/integration/runtime_capabilities_pending_test.go @@ -21,6 +21,7 @@ func TestManagedCapabilitiesWaitBeforeInitializationClaim(t *testing.T) { t.Fatal(err) } s := NewWithCredentialCipher(pool, cipher) + key := webDeployment(t, s, "e2b") tenant := uuid.NewString() session, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{ Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), @@ -35,8 +36,7 @@ func TestManagedCapabilitiesWaitBeforeInitializationClaim(t *testing.T) { t.Fatal(err) } provider := &initializingProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}, initializationPeer: initializationPeer{deferred: true}} - key := uuid.NewString() - worker, _ := managedWorkerMode(t, s, key, provider, false, true) + worker, _ := managedWorkerMode(t, s, key, provider, true) owner, err := worker.ProvisionEnvironment(t.Context(), tenant, env.ID, key) if err != nil || initializationState(t, s, owner.TenantID, owner.EnvironmentID) != "pending" { t.Fatal(owner, err) diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index c6949a6de..73f5e29d6 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -17,7 +17,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto/prototest" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -261,6 +260,7 @@ type computeLifecycleFixture struct { worker *execution.Worker stop func() key string + node string policy execution.RuntimeSuspensionPolicy } @@ -280,28 +280,21 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: uuid.NewString(), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour, MaxActive: maxActive, MaxRetained: maxRetained}} + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{IdleTimeout: time.Second, Retention: time.Hour}} + view, err := deploymentService(t, s).View(t.Context()) + if err != nil { + t.Fatal(err) + } + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID f.start() return f } func (f *computeLifecycleFixture) start() { t := f.t t.Helper() - dispatcher := &execution.Dispatcher{Registry: f.provider.registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: f.provider, Suspension: &f.policy}} - // Closing the previous Worker's connection can return before PostgreSQL drops its advisory lock. - deadline := time.Now().Add(2 * time.Second) - var w *execution.Worker - var err error - for { - w, err = startWorkerErr(t.Context(), f.store, dispatcher) - if err == nil || !errors.Is(err, pgunit.ErrLeaseHeld) || !time.Now().Before(deadline) { - break - } - time.Sleep(20 * time.Millisecond) - } - if err != nil { - t.Fatal(err) - } + w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) + // The Worker's claim starts a new owner epoch, in which the node reconnects. + onlineManagerNode(t, f.store, f.node) var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -521,47 +514,3 @@ func TestRuntimeComputeLifecycleSuspendedDeletionAndExpiryCleanup(t *testing.T) }) } } - -func TestRuntimeComputeLifecycleCapacityBoundsActiveAndRetained(t *testing.T) { - f := newComputeLifecycleFixture(t, 1, 2) - tenant, _, env, owner := f.create() - tenant2, _, env2 := managedSession(t, f.store) - if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatalf("active capacity ignored: %v", err) - } - if f.provider.creates != 1 { - t.Fatal("capacity rejection allocated compute") - } - f.complete(owner) - f.phase(tenant, env.ID, "suspended") - second, err := f.worker.ProvisionEnvironment(t.Context(), tenant2, env2.ID, f.key) - if err != nil { - t.Fatal(err) - } - pending := f.queued(owner) - for range 4 { - f.worker.ReconcileManagedRuntimes(t.Context()) - } - first, err := deploymentStore(f.store).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) - if err != nil || first.ComputePhase != "suspended" || f.provider.restores != 0 { - t.Fatal("wake exceeded active capacity", err) - } - f.sql(`UPDATE turns SET status='cancelled',completed_at=clock_timestamp() WHERE id=$1`, pending) - f.provider.mu.Lock() - b := f.provider.bootstraps[second.ID] - f.provider.mu.Unlock() - if err := f.provider.connect(t.Context(), b); err != nil { - t.Fatal(err) - } - second = f.phase(tenant2, env2.ID, "running") - f.complete(second) - f.phase(tenant2, env2.ID, "suspended") - // Both retained allocations count even when their source VMs are gone. - tenant3, _, env3 := managedSession(t, f.store) - if _, err := f.worker.ProvisionEnvironment(t.Context(), tenant3, env3.ID, f.key); !errors.Is(err, execution.ErrExecutionUnavailable) { - t.Fatalf("retained capacity ignored: %v", err) - } - if f.provider.creates != 2 { - t.Fatal("retained limit created a third allocation") - } -} diff --git a/services/core/tests/integration/runtime_configuration_cleanup_test.go b/services/core/tests/integration/runtime_configuration_cleanup_test.go index 2196f7674..c08993908 100644 --- a/services/core/tests/integration/runtime_configuration_cleanup_test.go +++ b/services/core/tests/integration/runtime_configuration_cleanup_test.go @@ -84,7 +84,7 @@ func TestManagedRuntimeConfigurationCleanup(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &configurationCleanupProvider{ lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: test.loseCreate}, rejectCreate: test.rejectCreate, settleCreate: test.settleCreate, diff --git a/services/core/tests/integration/runtime_connection_test.go b/services/core/tests/integration/runtime_connection_test.go index 19947a340..7bc66fb03 100644 --- a/services/core/tests/integration/runtime_connection_test.go +++ b/services/core/tests/integration/runtime_connection_test.go @@ -22,6 +22,7 @@ import ( func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, environment := managedSession(t, s) server := httptest.NewUnstartedServer(nil) wsURL := "ws://" + server.Listener.Addr().String() + "/api/v1/agent-daemon/ws" @@ -33,11 +34,7 @@ func TestManagedRuntimeConnectionTracksAuthenticatedSocket(t *testing.T) { server.Start() t.Cleanup(func() { server.Close(); runtime.CloseConnections(registry) }) p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - key := uuid.NewString() - start := func() *execution.Worker { - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: server.URL + "/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) - return w - } + start := func() *execution.Worker { return startWebWorker(t, s, registry, key, p, nil) } stop := func(w *execution.Worker) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/runtime_creation_settlement_test.go b/services/core/tests/integration/runtime_creation_settlement_test.go index b827e9c49..54390356d 100644 --- a/services/core/tests/integration/runtime_creation_settlement_test.go +++ b/services/core/tests/integration/runtime_creation_settlement_test.go @@ -42,7 +42,7 @@ func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { for _, cancelled := range []bool{false, true} { t.Run(map[bool]string{false: "live caller", true: "cancelled caller"}[cancelled], func(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) @@ -80,7 +80,7 @@ func TestManagedRuntimeConfirmedAbsentCreateReleasesAtomically(t *testing.T) { } func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{foreign: true} w, _ := managedWorker(t, s, key, p) tenant, _, environment := managedSession(t, s) @@ -94,7 +94,7 @@ func TestManagedRuntimeForeignAbsenceCannotReleaseCreation(t *testing.T) { } func TestManagedRuntimeObservedSettlementAllowsOwnedCleanup(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") p := &absentCreationProvider{observeSettled: true} w, _ := managedWorker(t, s, key, p) tenant, session, environment := managedSession(t, s) diff --git a/services/core/tests/integration/runtime_deployment_test.go b/services/core/tests/integration/runtime_deployment_test.go index 28d6bb24c..49cd6e6ac 100644 --- a/services/core/tests/integration/runtime_deployment_test.go +++ b/services/core/tests/integration/runtime_deployment_test.go @@ -1,12 +1,9 @@ package integration import ( - "context" "encoding/json" "errors" - "strings" "testing" - "time" "github.com/google/uuid" @@ -16,290 +13,94 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func deploymentSelection() deployment.ProcessDeployment { - return deployment.ProcessDeployment{InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("a", 64)} -} - -func deploymentConfigure(t *testing.T, w *Store, config *deployment.ProcessDeployment) { - t.Helper() - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), config); err != nil { - t.Fatal(err) - } - if config != nil && config.ProviderKind != "" { - legacyRuntimeSpecification(t, w, config.ProviderKind) - } -} - -// Lower-level legacy fixtures supply verified metadata without replaying the -// configuration transition being tested. Web setup owns this write in production. -func legacyRuntimeSpecification(t *testing.T, w *Store, provider string) { - t.Helper() - spec := SandboxDeploymentTestSpec(provider) - raw, _ := json.Marshal(spec) - if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification=$1", raw); err != nil { - t.Fatal(err) - } - if _, err := w.pool.Exec(t.Context(), "UPDATE runtime_nodes SET deployment_generation=1,ready_generation=1,specification_digest=$1", spec.Digest(provider)); err != nil { - t.Fatal(err) - } -} - -func TestRuntimeDeploymentPendingSessionsCannotMigrate(t *testing.T) { +// An installation never adopts hosted work admitted before it claimed the +// deployment: a pending Session or an unreleased allocation refuses the claim. +func TestRuntimeDeploymentClaimAdoptsNoUnclaimedWork(t *testing.T) { s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - tenant := uuid.NewString() - session, _ := localEnvironment(t, s, tenant) - old := deploymentSelection() - // First selection is allowed for work that has never had an installation. - deploymentConfigure(t, w, &old) - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 pending hosted") { - t.Fatal("pending Session migrated", err) + changes := deploymentExecution(t, executionWriter(t, s)) + installation, tenant := uuid.NewString(), uuid.NewString() + pending, _ := localEnvironment(t, s, tenant) + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("pending Session adopted", err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { - t.Fatal("pending Session orphaned by removing provider") - } - if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { + if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: pending.ID}); err != nil { t.Fatal(err) } - deploymentConfigure(t, w, &next) -} - -func TestRuntimeDeploymentUnknownAllocationsBlockAdoptionAndSwitch(t *testing.T) { - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - tenant := uuid.NewString() session, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) + owner, err := changes.ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())) if err != nil { t.Fatal(err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil || !strings.Contains(err.Error(), "no verified backend identity") { - t.Fatal("legacy allocation silently adopted", err) + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("unclaimed allocation adopted", err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: session.ID}); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).RequestCleanup(t.Context(), owner); err != nil { + if _, err := changes.RequestCleanup(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := changes.ReleaseAllocation(t.Context(), owner); !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal("unknown creation lost cleanup ownership", err) } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &old); err == nil { - t.Fatal("deleted unknown allocation did not block adoption") + if err := changes.Claim(t.Context(), installation); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("deleted unknown allocation did not block adoption", err) } - if _, err := deploymentExecution(t, w).SettleCreation(t.Context(), owner); err != nil { + if _, err := changes.SettleCreation(t.Context(), owner); err != nil { t.Fatal(err) } - if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), owner); err != nil { + if _, err := changes.ReleaseAllocation(t.Context(), owner); err != nil { t.Fatal(err) } - deploymentConfigure(t, w, &old) - _, environment = localEnvironment(t, s, tenant) - owner, err = deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { + if err := changes.Claim(t.Context(), installation); err != nil { t.Fatal(err) } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("unknown creation did not block switch", err) - } - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), nil); err == nil { - t.Fatal("removing adapter orphaned unknown creation") - } - replay, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil || !replay.Replayed || replay.ID != owner.ID { - t.Fatal("maintenance blocked receipt replay", replay, err) - } - if _, err := deploymentExecution(t, w).RequestCleanup(t.Context(), owner); err != nil { - t.Fatal("maintenance blocked cleanup", err) + if err := changes.RequireUnclaimed(t.Context()); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("owner without runtimes accepted a claimed deployment", err) } } -func TestRuntimeDeploymentMaintenancePreservesCreationRetriesAndOtherPlacements(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - deploymentConfigure(t, w, &old) +func TestRuntimeDeploymentResetPreservesCreationRetriesAndOtherPlacements(t *testing.T) { + s, w, installation := managedArchiveFixture(t) tenant := uuid.NewString() input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`)} existing, err := s.CreateSession(t.Context(), tenant, input) if err != nil { t.Fatal(err) } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) + ctx := SandboxResetTestContext(t.Context()) + if _, err := startReset(t, ctx, w, installation, deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } replay, err := s.CreateSession(t.Context(), tenant, input) if err != nil || replay.ID != existing.ID { t.Fatal("creation retry lost identity", err) } input.IdempotencyKey = uuid.NewString() - if _, err := s.CreateSession(t.Context(), tenant, input); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance created hosted Session", err) + if _, err := s.CreateSession(t.Context(), tenant, input); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset created hosted Session", err) } var count int - if err := pool.QueryRow(t.Context(), "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 1 { + if err := s.pool.QueryRow(t.Context(), "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 1 { t.Fatal("rejection left partial Session", count, err) } - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance reserved new allocation", err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset reserved new allocation", err) } for _, kind := range []string{"none", "self_hosted"} { input.IdempotencyKey = uuid.NewString() input.Configuration = json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"` + kind + `"}}`) if _, err := s.CreateSession(t.Context(), tenant, input); err != nil { - t.Fatal("maintenance blocked unrelated placement", kind, err) + t.Fatal("reset blocked unrelated placement", kind, err) } } - old.AdmissionPaused = false - deploymentConfigure(t, w, &old) - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("wrong installation reserved resource", err) - } - if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())); err != nil { - t.Fatal("resume did not reopen allocation", err) - } -} - -func TestRuntimeDeploymentMaintenanceSerializesHostedCreation(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - config := deploymentSelection() - deploymentConfigure(t, w, &config) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - tx, err := pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(context.Background()) - var blocker int32 - if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { - t.Fatal(err) - } - done := make(chan error, 1) - tenant := uuid.NewString() - go func() { - _, err := s.CreateSession(ctx, tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`)}) - done <- err - }() - runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, done) - if _, err := tx.Exec(ctx, "UPDATE runtime_deployment SET admission_paused=true"); err != nil { - t.Fatal(err) - } - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - if err := <-done; !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("creation bypassed committed maintenance", err) - } - var count int - if err := pool.QueryRow(ctx, "SELECT count(*) FROM sessions WHERE tenant_id=$1", tenant).Scan(&count); err != nil || count != 0 { - t.Fatal("racing creation left partial work", count, err) - } -} - -func TestRuntimeDeploymentRetainedResourcesBlockSwitchWithoutMutation(t *testing.T) { - for _, state := range []string{"running", "suspended", "cleanup_pending"} { - t.Run(state, func(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - old := deploymentSelection() - deploymentConfigure(t, w, &old) - tenant := uuid.NewString() - _, environment := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, old.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - if state == "suspended" { - _, err = pool.Exec(t.Context(), `UPDATE runtime_allocations SET state='running',create_settled=true,compute_phase='suspended',compute_state='{"snapshot":{"id":"retained-test-snapshot"}}' WHERE id=$1`, owner.ID) - } else { - _, err = pool.Exec(t.Context(), "UPDATE runtime_allocations SET state=$2,create_settled=true WHERE id=$1", owner.ID, state) - } - if err != nil { - t.Fatal(err) - } - old.AdmissionPaused = true - deploymentConfigure(t, w, &old) - var before, after, oldIdentity, newIdentity string - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&before); err != nil { - t.Fatal(err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&oldIdentity); err != nil { - t.Fatal(err) - } - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("retained resource allowed switch", state, err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(a)::text FROM runtime_allocations a WHERE id=$1", owner.ID).Scan(&after); err != nil { - t.Fatal(err) - } - if err := pool.QueryRow(t.Context(), "SELECT to_jsonb(d)::text FROM runtime_deployment d").Scan(&newIdentity); err != nil { - t.Fatal(err) - } - if before != after || oldIdentity != newIdentity { - t.Fatal("refused switch changed existing ownership") - } - }) - } -} - -func TestRuntimeDeploymentAllocationBeforeMaintenanceRetainsOwnership(t *testing.T) { - s, pool := newManagedTestStore(t) - w := executionWriter(t, s) - config := deploymentSelection() - deploymentConfigure(t, w, &config) - tenant := uuid.NewString() - _, environment := localEnvironment(t, s, tenant) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - tx, err := pool.Begin(ctx) - if err != nil { - t.Fatal(err) - } - defer tx.Rollback(context.Background()) - var blocker int32 - if err := tx.QueryRow(ctx, "SELECT pg_backend_pid() FROM runtime_deployment FOR UPDATE").Scan(&blocker); err != nil { + if err := deploymentExecution(t, w).CancelReset(ctx, installation, 1); err != nil { t.Fatal(err) } - allocated := make(chan error, 1) - go func() { - _, err := deploymentExecution(t, w).ReserveAllocation(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}, config.InstallationID, runtimedevice.HashCredential(uuid.NewString())) - allocated <- err - }() - runtimeSuspensionWaitBlocked(t, ctx, pool, blocker, allocated) - maintaining := make(chan error, 1) - maintenance := config - maintenance.AdmissionPaused = true - changes := deploymentExecution(t, w) - go func() { maintaining <- changes.ConfigureProcess(ctx, &maintenance) }() - if err := tx.Commit(ctx); err != nil { - t.Fatal(err) - } - if err := <-allocated; err != nil { - t.Fatal("earlier allocation lost ownership", err) - } - if err := <-maintaining; err != nil { - t.Fatal(err) - } - owner, err := deploymentStore(w).EnvironmentAllocation(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment.ID}) - if err != nil || owner.State != "creating" || owner.CreateSettled { - t.Fatal("maintenance changed uncertain receipt", owner, err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, uuid.NewString(), runtimedevice.HashCredential(uuid.NewString())); !errors.Is(err, placement.ErrAdmissionClosed) { + t.Fatal("wrong installation reserved resource", err) } - next := deploymentSelection() - next.AdmissionPaused = true - if err := deploymentExecution(t, w).ConfigureProcess(ctx, &next); err == nil || !strings.Contains(err.Error(), "1 unreleased allocations") { - t.Fatal("earlier in-flight allocation omitted from switch guard", err) + if _, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: existing.Environment.ID}, installation, runtimedevice.HashCredential(uuid.NewString())); err != nil { + t.Fatal("cancelled reset did not reopen allocation", err) } } diff --git a/services/core/tests/integration/runtime_deployment_worker_test.go b/services/core/tests/integration/runtime_deployment_worker_test.go index 42ce2fae8..6e852099f 100644 --- a/services/core/tests/integration/runtime_deployment_worker_test.go +++ b/services/core/tests/integration/runtime_deployment_worker_test.go @@ -1,7 +1,7 @@ package integration import ( - "strings" + "errors" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" @@ -13,7 +13,7 @@ import ( func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) { s, _ := newManagedTestStore(t) - key := uuid.NewString() + key := webDeployment(t, s, "e2b") old := &lifecycleProvider{resources: map[string]sandbox.Info{}} worker, stop := managedWorker(t, s, key, old) tenant, _, environment := managedSession(t, s) @@ -23,25 +23,15 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) } stop() replacement := &lifecycleProvider{resources: map[string]sandbox.Info{}} - config := &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: uuid.NewString(), BackendFingerprint: strings.Repeat("b", 64), Provider: replacement, AdmissionPaused: true} - start := func(config *execution.RuntimeProvider) error { - _, err := startWorkerErr(t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: config}) + start := func(runtimes *execution.RuntimeProvider) error { + _, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: runtimes}) return err } - if err := start(config); err == nil || !strings.Contains(err.Error(), "maintenance") { - t.Fatal("startup switched active deployment", err) + if err := start(webRuntimes(t, s, uuid.NewString(), replacement, nil)); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("startup switched the claimed installation", err) } - worker, stop = managedWorkerMode(t, s, key, old, true) - replay, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) - if err != nil || !replay.Replayed || replay.ID != owner.ID { - t.Fatal("maintenance interrupted existing allocation", replay, err) - } - stop() - if err := start(config); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { - t.Fatal("startup abandoned retained allocation", err) - } - if err := start(nil); err == nil || !strings.Contains(err.Error(), "unreleased allocations") { - t.Fatal("omitted configuration abandoned deployment", err) + if err := start(nil); !errors.Is(err, deployment.ErrConflict) { + t.Fatal("startup without runtimes abandoned the claimed deployment", err) } if replacement.creates != 0 || replacement.kills != 0 { t.Fatal("rejected startup touched new backend") @@ -51,6 +41,10 @@ func TestManagedDeploymentStartupRejectsSwitchBeforeBackendAccess(t *testing.T) t.Fatal("rejected startup rewrote resource owner", got, err) } // Failed startup relinquishes its lease, so the original backend can resume. - _, stop = managedWorker(t, s, key, old) + worker, stop = managedWorker(t, s, key, old) + replay, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key) + if err != nil || !replay.Replayed || replay.ID != owner.ID { + t.Fatal("rejected startup interrupted the existing allocation", replay, err) + } stop() } diff --git a/services/core/tests/integration/runtime_initialization_test.go b/services/core/tests/integration/runtime_initialization_test.go index 57a4bf5cc..8acc75925 100644 --- a/services/core/tests/integration/runtime_initialization_test.go +++ b/services/core/tests/integration/runtime_initialization_test.go @@ -52,6 +52,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { t.Fatal(err) } s := NewWithCredentialCipher(pool, cipher) + key := webDeployment(t, s, "e2b") tenant := uuid.NewString() input := sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"environment":{"type":"openai_hosted"}}`), InitialFiles: []environmentconfig.InitialFile{{Type: "inline", Path: "/workspace/a", Data: []byte("first")}, {Type: "inline", Path: "/workspace/b", Data: []byte("second")}}} if setupOnly { @@ -85,8 +86,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } return completedInitialization(proto.RuntimePreparePayload{}, nil) } - key := uuid.NewString() - w, stop := managedWorkerMode(t, s, key, p, false, true) + w, stop := managedWorkerMode(t, s, key, p, true) if mode == "restart" { awaitInitialization(t, s, tenant, env.ID, "failed") if p.writes.Load() != 0 { @@ -122,7 +122,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { t.Fatal("completed preparation blocked", err) } stop() - _, _ = managedWorkerMode(t, s, key, p, false, true) + _, _ = managedWorkerMode(t, s, key, p, true) time.Sleep(350 * time.Millisecond) if int(p.writes.Load()) != expectedSteps { t.Fatal("completed preparation replayed") @@ -141,7 +141,7 @@ func TestEnvironmentInitializationCompletionUnknownAndRestart(t *testing.T) { } func TestManagedRuntimePreparationAllOperationsUsePeer(t *testing.T) { - s := hostedFailureStore(t) + s, key := hostedFailureStore(t) var archive bytes.Buffer writer := zip.NewWriter(&archive) for path, body := range map[string]string{"proof/.codex-plugin/plugin.json": `{"name":"plugin","description":"A plugin.","skills":"./skills"}`, "proof/skills/example/SKILL.md": "---\nname: plugin-proof\ndescription: A plugin Skill.\n---\nProof."} { @@ -181,8 +181,7 @@ func TestManagedRuntimePreparationAllOperationsUsePeer(t *testing.T) { actionsMu.Unlock() return completedInitialization(request, data) } - key := uuid.NewString() - worker, _ := managedWorkerMode(t, s, key, provider, false, true) + worker, _ := managedWorkerMode(t, s, key, provider, true) if _, err := worker.ProvisionEnvironment(t.Context(), tenant, environment.ID, key); err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/runtime_input_admission_test.go b/services/core/tests/integration/runtime_input_admission_test.go index 8c45dac30..725bb0567 100644 --- a/services/core/tests/integration/runtime_input_admission_test.go +++ b/services/core/tests/integration/runtime_input_admission_test.go @@ -7,13 +7,15 @@ import ( "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) -func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { +func TestManagedRuntimeResetPreservesCancelAndRetry(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, _ := managedSession(t, s) inputs := []sessions.Input{messageInput("accepted work")} accepted, err := submitInputs(t.Context(), s, tenant, session.ID, "work", inputs) @@ -21,10 +23,13 @@ func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { t.Fatal(err) } p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - w, stop := managedWorkerMode(t, s, uuid.NewString(), p, true) + w, stop := managedWorker(t, s, key, p) defer stop() - if _, err := w.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, placement.ErrAdmissionClosed) { - t.Fatal("maintenance accepted new hosted Session", err) + if _, err := w.StartSandboxReset(SandboxResetTestContext(t.Context()), deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { + t.Fatal(err) + } + if _, err := w.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: session.Configuration}); !errors.Is(err, placement.ErrResetAdmission) { + t.Fatal("reset accepted new hosted Session", err) } cancel := []sessions.Input{{Kind: "cancel", Payload: json.RawMessage(`{}`)}} first, err := w.SubmitInputs(t.Context(), tenant, session.ID, "cancel", cancel) @@ -40,7 +45,7 @@ func TestManagedRuntimeMaintenancePreservesCancelAndRetry(t *testing.T) { t.Fatal("matching input retry lost its accepted outcome", retry, err) } if _, err := w.SubmitInputs(t.Context(), uuid.NewString(), session.ID, "cancel", cancel); !errors.Is(err, sessions.ErrNotFound) { - t.Fatal("maintenance weakened tenant isolation", err) + t.Fatal("reset weakened tenant isolation", err) } if p.creates != 0 { t.Fatal("existing controls provisioned a new Runtime") diff --git a/services/core/tests/integration/runtime_lifecycle_nodes_test.go b/services/core/tests/integration/runtime_lifecycle_nodes_test.go index a02c00452..72f26498a 100644 --- a/services/core/tests/integration/runtime_lifecycle_nodes_test.go +++ b/services/core/tests/integration/runtime_lifecycle_nodes_test.go @@ -37,7 +37,7 @@ func lifecycleTestSession(t *testing.T, s *Store, node string) (string, sessions } return tenant, session } -func lifecycleTestAllocation(t *testing.T, s, w *Store, d deployment.ProcessDeployment, node string) deployment.Allocation { +func lifecycleTestAllocation(t *testing.T, s, w *Store, d managerNode, node string) deployment.Allocation { t.Helper() tenant, session := lifecycleTestSession(t, s, node) allocation, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}, d.InstallationID, runtimedevice.HashCredential(uuid.NewString())) @@ -52,20 +52,20 @@ func TestRuntimeLifecycleNodePagesAreIndependent(t *testing.T) { other := lifecycleTestNode(t, s) var allocated, pending []string for range 34 { - allocated = append(allocated, lifecycleTestAllocation(t, s, w, d, d.LocalNodeID).ID) - _, session := lifecycleTestSession(t, s, d.LocalNodeID) + allocated = append(allocated, lifecycleTestAllocation(t, s, w, d, d.NodeID).ID) + _, session := lifecycleTestSession(t, s, d.NodeID) pending = append(pending, session.Environment.ID) } second := lifecycleTestAllocation(t, s, w, d, other) _, secondPending := lifecycleTestSession(t, s, other) // Offline and unresolved cleanup remain discoverable without changing placement. - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET state='cleanup_pending' WHERE node_id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET state='cleanup_pending' WHERE node_id=$1", d.NodeID); err != nil { t.Fatal(err) } - for _, node := range []string{d.LocalNodeID, other} { + for _, node := range []string{d.NodeID, other} { var gotAlloc, gotPending []string cursor := "" for range 4 { @@ -181,7 +181,7 @@ func TestRuntimeLifecycleNodeInventoryAndRouting(t *testing.T) { t.Fatal(err) } nodes, err = deploymentStore(w).LifecycleNodes(t.Context()) - if err != nil || len(nodes) != 1 || nodes[0] != d.LocalNodeID { + if err != nil || len(nodes) != 1 || nodes[0] != d.NodeID { t.Fatal("removed node discovered", nodes, err) } } @@ -190,14 +190,14 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { for _, mutation := range []string{"DELETE FROM runtime_placements WHERE environment_id=$1", "UPDATE runtime_placements SET released_at=clock_timestamp() WHERE environment_id=$1"} { t.Run(mutation[:6], func(t *testing.T) { s, w, d := managerFixture(t, 4, 4) - tenant, session := lifecycleTestSession(t, s, d.LocalNodeID) + tenant, session := lifecycleTestSession(t, s, d.NodeID) if _, err := s.pool.Exec(t.Context(), mutation, session.Environment.ID); err != nil { t.Fatal(err) } if _, err := deploymentService(t, w).LifecycleNode(t.Context(), tenant, session.Environment.ID); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("invalid placement routed", err) } - rows, err := deploymentStore(w).UnallocatedEnvironments(t.Context(), d.LocalNodeID, "") + rows, err := deploymentStore(w).UnallocatedEnvironments(t.Context(), d.NodeID, "") if err != nil || len(rows) != 0 { t.Fatal("invalid placement provisioned", rows, err) } @@ -206,7 +206,12 @@ func TestRuntimeLifecycleNodeRejectsMissingOrReleasedPlacement(t *testing.T) { } func TestRuntimeLifecycleNodelessLane(t *testing.T) { - s, w, _, a := nodelessAllocationFixture(t) + s, w, installation := managedArchiveFixture(t) + _, reserved := localEnvironment(t, s, uuid.NewString()) + a, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: reserved.TenantID, EnvironmentID: reserved.ID}, installation, runtimedevice.HashCredential("runtime")) + if err != nil { + t.Fatal(err) + } nodes, err := deploymentStore(w).LifecycleNodes(t.Context()) if err != nil || !slices.Equal(nodes, []string{""}) { t.Fatal(nodes, err) diff --git a/services/core/tests/integration/runtime_lifecycle_test.go b/services/core/tests/integration/runtime_lifecycle_test.go index f03f85e0a..8da737e4a 100644 --- a/services/core/tests/integration/runtime_lifecycle_test.go +++ b/services/core/tests/integration/runtime_lifecycle_test.go @@ -72,12 +72,15 @@ func (p *lifecycleProvider) RunCommand(context.Context, sandbox.Reference, sandb return sandbox.CommandResult{}, errors.New("not used") } +// managedWorker starts a Worker that runs the Web setup webDeployment +// committed for installation key on p. func managedWorker(t *testing.T, s *Store, key string, p sandbox.SandboxProvider) (*execution.Worker, func()) { t.Helper() return managedWorkerMode(t, s, key, p, false) } -func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProvider, maintenance bool, run ...bool) (*execution.Worker, func()) { +// managedWorkerMode is managedWorker that also runs the Worker when run is set. +func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProvider, run bool) (*execution.Worker, func()) { t.Helper() registry := runtimegateway.NewRegistry() if peer, ok := p.(interface { @@ -88,8 +91,8 @@ func managedWorkerMode(t *testing.T, s *Store, key string, p sandbox.SandboxProv t.Cleanup(server.Close) peer.setRuntimeGateway(t, "ws"+strings.TrimPrefix(server.URL, "http"), registry) } - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p, AdmissionPaused: maintenance}}) - if len(run) > 0 && run[0] { + w := startWebWorker(t, s, registry, key, p, nil) + if run { ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) go func() { done <- w.Run(ctx) }() @@ -146,8 +149,8 @@ func reconcileManagedState(t *testing.T, w *execution.Worker, s *Store, tenant, func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true} w, stop := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) @@ -185,8 +188,8 @@ func TestManagedRuntimeLostCreateRestartAndDeletion(t *testing.T) { func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, session, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}, loseCreate: true, absent: true} w, _ := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) @@ -213,37 +216,10 @@ func TestManagedRuntimeUnknownCreationRetainsCleanup(t *testing.T) { } } -func TestManagedRuntimeExpiryRevokesWhenProviderUnavailable(t *testing.T) { - s, _ := newManagedTestStore(t) - tenant, _, env := managedSession(t, s) - key := uuid.NewString() - p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - w, _ := managedWorker(t, s, key, p) - owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) - if err != nil { - t.Fatal(err) - } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '61 minutes' WHERE id=$1", owner.ID); err != nil { - t.Fatal(err) - } - p.unavailable = true - reconcileManagedState(t, w, s, tenant, env.ID, "cleanup_pending") - got, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}) - if err != nil || got.State != "cleanup_pending" { - t.Fatalf("expiry lost on provider failure: %+v %v", got, err) - } - if _, ok, err := sessionAdapter(s).GetDeviceCredential(t.Context(), owner.DeviceID); err != nil || ok { - t.Fatal("expired credential still authenticates") - } - if p.kills != 0 { - t.Fatal("unavailable provider misreported cleanup") - } -} - func TestManagedRuntimeStoppedComputeDoesNotRequestCleanup(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, _, env := managedSession(t, s) - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} w, _ := managedWorker(t, s, key, p) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) diff --git a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go index d581b19f6..771c4c99f 100644 --- a/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go +++ b/services/core/tests/integration/runtime_node_lifecycle_fixture_test.go @@ -18,7 +18,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmentconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimedevice" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -124,21 +123,13 @@ func newNodeIsolationFixture(t *testing.T, mode string) *nodeIsolationFixture { cp.mu.Unlock() server.Close() }) - f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: uuid.NewString(), nodeA: uuid.NewString(), nodeB: uuid.NewString()} + f := &nodeIsolationFixture{initializationCancel: cancelPreparation, t: t, store: s, nodes: deploymentService(t, s), pool: pool, provider: p, key: webDeployment(t, s, "microsandbox"), nodeA: uuid.NewString(), nodeB: uuid.NewString()} // Keep restored compute awake throughout the isolation assertions. // The suspension setup explicitly dates its activity two minutes in the past. - policy := &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour, MaxActive: 100, MaxRetained: 100} - f.worker = startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, BackendFingerprint: strings.Repeat("a", 64), Provider: p, ProviderKind: "microsandbox", LocalNodeID: f.nodeA, LocalCredentialSHA256: runtimedevice.HashCredential("local-credential"), LocalMaxActive: 100, LocalMaxRetained: 100, Suspension: policy}}) - spec := SandboxDeploymentTestSpec("microsandbox") - raw, _ := json.Marshal(spec) - if _, err := pool.Exec(t.Context(), "UPDATE runtime_deployment SET specification=$1", raw); err != nil { - t.Fatal(err) - } - if _, err := pool.Exec(t.Context(), "UPDATE runtime_nodes SET specification_digest=$1,deployment_generation=1", spec.Digest("microsandbox")); err != nil { - t.Fatal(err) - } + f.worker = startWebWorker(t, s, registry, f.key, p, &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Minute, Retention: time.Hour}) t.Cleanup(f.stop) f.epoch = fixtureOwnerEpoch(t, s) + f.enroll(f.nodeA) f.enroll(f.nodeB) f.online(f.nodeA) f.online(f.nodeB) diff --git a/services/core/tests/integration/runtime_nodes_test.go b/services/core/tests/integration/runtime_nodes_test.go index cf11c0a42..c00d88ceb 100644 --- a/services/core/tests/integration/runtime_nodes_test.go +++ b/services/core/tests/integration/runtime_nodes_test.go @@ -17,19 +17,15 @@ import ( "github.com/jackc/pgx/v5" ) -func managerFixture(t *testing.T, active, retained int) (*Store, *Store, deployment.ProcessDeployment) { +// managerNode is the deployment managerFixture sets up: installation +// InstallationID runs Docker on the one online node NodeID. +type managerNode struct{ InstallationID, NodeID string } + +func managerFixture(t *testing.T, active, retained int) (*Store, *Store, managerNode) { t.Helper() - s, _ := newManagedTestStore(t) - w := executionWriter(t, s) - d := deploymentSelection() - d.ProviderKind = "docker" - d.LocalNodeID = uuid.NewString() - d.LocalCredentialSHA256 = runtimedevice.HashCredential("local-node-credential") - d.LocalMaxActive = active - d.LocalMaxRetained = retained - deploymentConfigure(t, w, &d) - onlineManagerNode(t, s, d.LocalNodeID) - return s, w, d + s, w, view, _ := webSpecificationFixture(t, "docker") + node := enrollNode(t, s, view, deployment.Capacity{MaxActive: active, MaxRetained: retained}) + return s, w, managerNode{InstallationID: view.InstallationID, NodeID: node.NodeID} } func onlineManagerNode(t *testing.T, s *Store, id string) string { t.Helper() @@ -158,7 +154,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { if err != nil || len(nodes) != 1 || nodes[0].Active != 1 || nodes[0].Retained != 1 || nodes[0].Reserved != 1 { t.Fatal(nodes, err) } - if err := service.RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := service.RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed pending placement", err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: retained.ID}); err != nil { @@ -169,7 +165,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } replay, err := s.CreateSession(t.Context(), tenant, input) @@ -180,7 +176,7 @@ func TestRuntimeNodesAtomicPlacementAndRetry(t *testing.T) { t.Fatal("foreign placement leaked", err) } placement, err := sessionRuntimePlacement(t.Context(), s, tenant, first.ID) - if err != nil || placement.NodeID != d.LocalNodeID || placement.Available { + if err != nil || placement.NodeID != d.NodeID || placement.Available { t.Fatal(placement, err) } } @@ -220,7 +216,9 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { } } epoch := managerEpoch(t, s) - deploymentConfigure(t, w, &d) + if err := deploymentExecution(t, w).Claim(t.Context(), d.InstallationID); err != nil { + t.Fatal(err) + } if next := managerEpoch(t, s); next != epoch+1 { t.Fatal(next) } @@ -230,7 +228,7 @@ func TestRuntimeNodesEnrollmentAndEpoch(t *testing.T) { if _, err := s.CreateSession(t.Context(), uuid.NewString(), managerSessionInput("stale")); !errors.Is(err, placement.ErrNodeUnavailable) { t.Fatal("stale node admitted", err) } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) if err := nodes.RemoveNode(t.Context(), input.NodeID); err != nil { t.Fatal(err) } @@ -254,7 +252,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal(err) } if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: pending.ID}); err != nil { @@ -267,7 +265,7 @@ func TestRuntimeNodesRetention(t *testing.T) { if err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := nodes.RemoveNode(t.Context(), next.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("unknown cleanup released node", err) } retained, err = deploymentExecution(t, w).SettleCreation(t.Context(), retained) @@ -277,22 +275,8 @@ func TestRuntimeNodesRetention(t *testing.T) { if _, err := deploymentExecution(t, w).ReleaseAllocation(t.Context(), retained); err != nil { t.Fatal(err) } - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); !errors.Is(err, deployment.ErrLocalNodeConfigured) { - t.Fatal("configured local node was removed", err) - } - if _, err := nodes.AuthenticateNode(t.Context(), next.LocalNodeID, "local-node-credential"); err != nil { - t.Fatal("rejected removal changed local credentials", err) - } - next.AdmissionPaused = true - deploymentConfigure(t, w, &next) - detached := next - detached.LocalNodeID = "" - detached.LocalCredentialSHA256 = "" - detached.LocalMaxActive, detached.LocalMaxRetained = 0, 0 - detached.BackendFingerprint = strings.Repeat("b", 64) - deploymentConfigure(t, w, &detached) - if err := nodes.RemoveNode(t.Context(), next.LocalNodeID); err != nil { - t.Fatal("detached resolved node cannot be removed", err) + if err := nodes.RemoveNode(t.Context(), next.NodeID); err != nil { + t.Fatal("released node cannot be removed", err) } } func TestRuntimeNodesRestoreAndCreationShareCapacity(t *testing.T) { @@ -371,22 +355,17 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET kept_at=clock_timestamp()-interval '2 days' WHERE id=$1", owner.ID); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } offline, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || offline.Expired || offline.State != "running" { t.Fatal("offline treated as destructive expiry", offline, err) } - if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("offline ownership discarded", err) } - changed := d - changed.LocalNodeID = uuid.NewString() - if err := deploymentExecution(t, w).ConfigureProcess(t.Context(), &changed); err == nil { - t.Fatal("lost local state created replacement identity") - } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) resumed, err := deploymentExecution(t, w).ObserveRunning(t.Context(), offline) if err != nil || resumed.ID != owner.ID || resumed.DeviceID != owner.DeviceID || resumed.NodeID != owner.NodeID { t.Fatal("reconnect changed instance", resumed, err) @@ -397,20 +376,20 @@ func TestRuntimeNodesLongOfflineRetainsExactAllocation(t *testing.T) { if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_phase='suspended',compute_state=$2::jsonb,compute_retained_until=clock_timestamp()+interval '1 day' WHERE id=$1", owner.ID, json.RawMessage(`{"snapshot":{"id":"same-snapshot"}}`)); err != nil { t.Fatal(err) } - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.LocalNodeID); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1", d.NodeID); err != nil { t.Fatal(err) } retained, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || retained.Expired || string(retained.ComputeState) != `{"snapshot": {"id": "same-snapshot"}}` { t.Fatal(retained, err) } - onlineManagerNode(t, s, d.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) same, err := deploymentStore(s).EnvironmentAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: session.Environment.ID}) if err != nil || same.ID != owner.ID || string(same.ComputeState) != string(retained.ComputeState) { t.Fatal("snapshot changed across reconnect", same, err) } placement, err := sessionRuntimePlacement(t.Context(), s, tenant, session.ID) - if err != nil || placement.NodeID != d.LocalNodeID { + if err != nil || placement.NodeID != d.NodeID { t.Fatal(placement, err) } if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_allocations SET compute_retained_until=clock_timestamp()-interval '1 second' WHERE id=$1", owner.ID); err != nil { diff --git a/services/core/tests/integration/runtime_observation_test.go b/services/core/tests/integration/runtime_observation_test.go index 0c178075c..5ad2acf2b 100644 --- a/services/core/tests/integration/runtime_observation_test.go +++ b/services/core/tests/integration/runtime_observation_test.go @@ -31,7 +31,7 @@ func TestRuntimeNodeObservationRetainsResourcesAndFencesStaleResults(t *testing. if err != nil || retained.State != "running" || retained.ID != owner.ID || retained.ObservationError != "node_unavailable" { t.Fatal(retained, err) } - if err := deploymentService(t, s).RemoveNode(t.Context(), d.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("diagnostic released resource", err) } // A new lifecycle observation must not be erased by an earlier result. diff --git a/services/core/tests/integration/runtime_pending_test.go b/services/core/tests/integration/runtime_pending_test.go index 8245750f5..06af06ef7 100644 --- a/services/core/tests/integration/runtime_pending_test.go +++ b/services/core/tests/integration/runtime_pending_test.go @@ -15,6 +15,7 @@ import ( func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) { s, _ := newManagedTestStore(t) + key := webDeployment(t, s, "e2b") tenant, idle, idleEnvironment := managedSession(t, s) initial, err := s.CreateSession(t.Context(), tenant, sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted"}}`), InitialInputs: []sessions.Input{messageInput("hello")}}) if err != nil { @@ -24,12 +25,8 @@ func TestManagedRuntimeAutomaticBootstrapRecoversCommittedSessions(t *testing.T) if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: deleted.TenantID, SessionID: deleted.ID}); err != nil { t.Fatal(err) } - key := uuid.NewString() p := &lifecycleProvider{resources: map[string]sandbox.Info{}} - start := func() *execution.Worker { - w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: &execution.RuntimeProvider{CoreURL: "http://core.invalid/api/v1", InstallationID: key, BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Provider: p}}) - return w - } + start := func() *execution.Worker { return startWebWorker(t, s, runtimegateway.NewRegistry(), key, p, nil) } stop := func(w *execution.Worker) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/services/core/tests/integration/runtime_scan_test.go b/services/core/tests/integration/runtime_scan_test.go index c5221dbc1..5f88de644 100644 --- a/services/core/tests/integration/runtime_scan_test.go +++ b/services/core/tests/integration/runtime_scan_test.go @@ -8,8 +8,6 @@ import ( "testing" "time" - "github.com/google/uuid" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" @@ -30,7 +28,7 @@ func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { t.Run(fmt.Sprint(count), func(t *testing.T) { s, _ := newManagedTestStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := uuid.NewString() + key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) var ids []string for range count { @@ -69,7 +67,7 @@ func TestManagedRuntimeScanWrapServicesNextPage(t *testing.T) { func TestManagedRuntimeScanEmptyAfterCleanupAndCanceledCall(t *testing.T) { s, _ := newManagedTestStore(t) p := &scanProvider{lifecycleProvider: lifecycleProvider{resources: map[string]sandbox.Info{}}} - key := uuid.NewString() + key := webDeployment(t, s, "e2b") w, _ := managedWorker(t, s, key, p) tenant, session, env := managedSession(t, s) owner, err := w.ProvisionEnvironment(t.Context(), tenant, env.ID, key) diff --git a/services/core/tests/integration/runtime_suspension_test.go b/services/core/tests/integration/runtime_suspension_test.go index 402338582..6ca4cd340 100644 --- a/services/core/tests/integration/runtime_suspension_test.go +++ b/services/core/tests/integration/runtime_suspension_test.go @@ -261,47 +261,6 @@ func TestRuntimeSuspensionRetentionAndDeletedSession(t *testing.T) { } } -func TestRuntimeSuspensionCountsUncertainCapacityUntilReleased(t *testing.T) { - s, pool := testStore(t) - w := executionWriter(t, s) - provider := uuid.NewString() - cases := []struct { - state, phase string - count bool - }{ - {"creating", "disabled", true}, {"running", "running", true}, {"running", "quiescing", true}, {"running", "suspending", true}, {"running", "suspended", false}, {"running", "restoring", true}, {"running", "waking", true}, {"cleanup_pending", "restoring", true}, {"released", "running", false}, - } - want := int64(0) - wantRetained := int64(0) - for _, item := range cases { - tenant := uuid.NewString() - _, env := localEnvironment(t, s, tenant) - owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: tenant, EnvironmentID: env.ID}, provider, runtimedevice.HashCredential(uuid.NewString())) - if err != nil { - t.Fatal(err) - } - runtimeSuspensionSQL(t, pool, `UPDATE runtime_allocations SET state=$2,compute_phase=$3,create_settled=($2<>'creating'),released_at=CASE WHEN $2='released' THEN clock_timestamp() END WHERE id=$1`, owner.ID, item.state, item.phase) - if item.count { - want++ - } - if item.state != "released" { - wantRetained++ - } - retained, err := deploymentStore(w).CountRetainedAllocations(t.Context(), provider) - if err != nil || retained != wantRetained { - t.Fatalf("retained capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, retained, wantRetained, err) - } - got, err := deploymentStore(w).CountComputeReservations(t.Context(), provider) - if err != nil || got != want { - t.Fatalf("capacity state=%s phase=%s got=%d want=%d err=%v", item.state, item.phase, got, want, err) - } - } - got, err := deploymentStore(w).CountComputeReservations(t.Context(), uuid.NewString()) - if err != nil || got != 0 { - t.Fatal("capacity crossed installation boundary", got, err) - } -} - func TestRuntimeSuspensionIdleStartsAfterLastCompletion(t *testing.T) { _, w, pool, owner := runtimeSuspensionFixture(t) turn := runtimeSuspensionCompleted(t, pool, owner) @@ -457,7 +416,7 @@ func TestRuntimeComputePhaseChangedAtInNodeAllocations(t *testing.T) { } listed := func() deployment.NodeAllocation { t.Helper() - items, err := deploymentStore(s).NodeAllocations(t.Context(), d.LocalNodeID) + items, err := deploymentStore(s).NodeAllocations(t.Context(), d.NodeID) if err != nil || len(items) != 1 || items[0].ID != allocation.ID { t.Fatal(items, err) } diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index 0d29a6720..aedeca68c 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -73,14 +73,8 @@ func newWakeHintIntegration(t *testing.T) *wakeHintIntegration { fakeCheckpointProvider: f.provider, sentinel: sentinel.owner.ID, release: make(chan struct{}), scans: make(chan int, 16), } - worker := startWorker(t, t.Context(), f.store, &execution.Dispatcher{ - Registry: f.provider.registry, - ManagedRuntimes: &execution.RuntimeProvider{ - CoreURL: "http://core.invalid/api/v1", InstallationID: f.key, - BackendFingerprint: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - Provider: provider, Suspension: &f.policy, - }, - }) + worker := startWebWorker(t, f.store, f.provider.registry, f.key, provider, &f.policy) + onlineManagerNode(t, f.store, f.node) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error, 1) var once sync.Once diff --git a/services/core/tests/integration/sandbox_deployment_switch_test.go b/services/core/tests/integration/sandbox_deployment_switch_test.go index 7c88d88c0..37b73464b 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_test.go @@ -190,7 +190,7 @@ func TestSandboxSwitchRetiresNodesAndEnrollment(t *testing.T) { if err := deploymentExecution(t, w).StartReset(SandboxResetTestContext(t.Context()), id, deployment.ResetRequest{Clear: "auto", ExpectedGeneration: 1}); err != nil { t.Fatal(err) } - // AdmissionPaused rejects a valid enrollment without consuming it. Authentication + // A reset rejects a valid enrollment without consuming it. Authentication // still precedes deployment details for invalid or retired credentials. spareNode := node spareNode.NodeID = uuid.NewString() diff --git a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go index e43dd7ae3..af2c98e9f 100644 --- a/services/core/tests/integration/sandbox_deployment_switch_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_switch_worker_test.go @@ -35,13 +35,13 @@ func TestSandboxWorkerSwitchesAndRecoversFailedActivation(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil + return &execution.RuntimeProvider{InstallationID: id, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}, nil }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { preparations.Add(1) if fail.Load() { return execution.PreparedRuntimeDeployment{}, errors.New("fixture provider unavailable") } - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) w := startWorker(t, t.Context(), s, &execution.Dispatcher{Registry: runtimegateway.NewRegistry(), ManagedRuntimes: configuration}) ctx, cancel := context.WithCancel(t.Context()) diff --git a/services/core/tests/integration/sandbox_deployment_worker_test.go b/services/core/tests/integration/sandbox_deployment_worker_test.go index 95538e64a..dcae13238 100644 --- a/services/core/tests/integration/sandbox_deployment_worker_test.go +++ b/services/core/tests/integration/sandbox_deployment_worker_test.go @@ -27,10 +27,10 @@ func TestSandboxDeploymentWorkerActivatesWithoutRestart(t *testing.T) { if err != nil || setup.Provider == "" { return nil, err } - return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, BackendFingerprint: setup.BackendFingerprint, CoreURL: "https://core.example/api/v1", Provider: p}, nil }, func(ctx context.Context, setup deployment.Setup) (execution.PreparedRuntimeDeployment, error) { - return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil + return execution.PreparedRuntimeDeployment{Config: &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, CoreURL: "https://core.example/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p}}, nil }) start := func() (*execution.Worker, func()) { t.Helper() diff --git a/services/core/tests/integration/sandbox_node_auth_order_http_test.go b/services/core/tests/integration/sandbox_node_auth_order_http_test.go index 67f9d9615..4e152b8a3 100644 --- a/services/core/tests/integration/sandbox_node_auth_order_http_test.go +++ b/services/core/tests/integration/sandbox_node_auth_order_http_test.go @@ -75,7 +75,7 @@ func TestSandboxNodeRoutesAuthenticateBeforeDeploymentState(t *testing.T) { // Once Web claims an installation, still before initialization, another // installation's token gets the same 401 it gets after initialization. - if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET installation_id=$1, web_managed=true WHERE singleton=true", claimed); err != nil { + if _, err := s.pool.Exec(t.Context(), "UPDATE runtime_deployment SET installation_id=$1 WHERE singleton=true", claimed); err != nil { t.Fatal(err) } run([]check{ diff --git a/services/core/tests/integration/sandbox_reset_test.go b/services/core/tests/integration/sandbox_reset_test.go index 43316c0e0..cc37abdc4 100644 --- a/services/core/tests/integration/sandbox_reset_test.go +++ b/services/core/tests/integration/sandbox_reset_test.go @@ -251,20 +251,17 @@ func TestSandboxResetAuditFailureRollsBackPauseAndCompletion(t *testing.T) { } func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { - s, w, process := managerFixture(t, 10, 10) - // Reuse the real placement fixture, then adopt its selection as Web-managed. - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_deployment SET web_managed=true,local_node_id=NULL`) - runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET deployment_generation=1,specification_digest=$1`, SandboxDeploymentTestSpec("docker").Digest("docker")) + s, w, d := managerFixture(t, 10, 10) _, pending := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) tenant, suspended := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - allocation := archiveAllocation(t, w, tenant, suspended, process.InstallationID) + allocation := archiveAllocation(t, w, tenant, suspended, d.InstallationID) runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_allocations SET compute_phase='suspended',compute_retained_until=clock_timestamp()+interval '1 hour' WHERE id=$1`, allocation.ID) tenant, deleted := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) - archiveAllocation(t, w, tenant, deleted, process.InstallationID) + archiveAllocation(t, w, tenant, deleted, d.InstallationID) if err := sessionService(t, s).DeleteSession(t.Context(), sessions.DeleteSessionCommand{TenantID: tenant, SessionID: deleted.ID}); err != nil { t.Fatal(err) } - reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, process.InstallationID, deployment.ResetRequest{ExpectedGeneration: 1, Clear: "auto"}) + reset, err := startReset(t, SandboxResetTestContext(t.Context()), w, d.InstallationID, deployment.ResetRequest{ExpectedGeneration: 1, Clear: "auto"}) if err != nil { t.Fatal(err) } @@ -274,7 +271,7 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { } for _, state := range []string{"preparing", "stale", "epoch", "disconnected"} { runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET connected_epoch=(SELECT owner_epoch FROM runtime_deployment)`) - onlineManagerNode(t, s, process.LocalNodeID) + onlineManagerNode(t, s, d.NodeID) switch state { case "preparing": runtimeSuspensionSQL(t, s.pool, `UPDATE runtime_nodes SET provider_ready=false`) @@ -298,14 +295,14 @@ func TestSandboxResetSnapshotCountsOfflineOwnershipOnce(t *testing.T) { if view.Reset.Remaining.OnOfflineNodes != want { t.Fatalf("%s presence: %+v", state, view.Reset.Remaining) } - if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != process.LocalNodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { + if want > 0 && (len(view.Reset.Remaining.OfflineNodes) != 1 || view.Reset.Remaining.OfflineNodes[0].NodeID != d.NodeID || view.Reset.Remaining.OfflineNodes[0].Resources != 3) { t.Fatal("offline ownership projection", view.Reset.Remaining) } } - if _, err := deploymentExecution(t, w).CompleteReset(t.Context(), process.InstallationID, 1, reset.Reset.RequestedAt); err == nil { + if _, err := deploymentExecution(t, w).CompleteReset(t.Context(), d.InstallationID, 1, reset.Reset.RequestedAt); err == nil { t.Fatal("offline resources were treated as cleaned") } - if err := deploymentService(t, s).RemoveNode(t.Context(), process.LocalNodeID); !errors.Is(err, deployment.ErrNodeInUse) { + if err := deploymentService(t, s).RemoveNode(t.Context(), d.NodeID); !errors.Is(err, deployment.ErrNodeInUse) { t.Fatal("removed node with reset resources", err) } if row, err := sessionAdapter(s).GetEnvironment(t.Context(), pending.TenantID, pending.Environment.ID); err == nil && row.Status == "expired" { diff --git a/services/core/tests/integration/sandbox_specification_lifecycle_test.go b/services/core/tests/integration/sandbox_specification_lifecycle_test.go index 19e9e2513..314e90758 100644 --- a/services/core/tests/integration/sandbox_specification_lifecycle_test.go +++ b/services/core/tests/integration/sandbox_specification_lifecycle_test.go @@ -43,9 +43,15 @@ func webSpecificationFixture(t *testing.T, provider string) (*Store, *Store, dep } func specificationNode(t *testing.T, s *Store, view deployment.View) deployment.Enrollment { + t.Helper() + return enrollNode(t, s, view, deployment.Capacity{MaxActive: 4, MaxRetained: 16}) +} + +// enrollNode enrolls an online node with capacity on the committed setup view. +func enrollNode(t *testing.T, s *Store, view deployment.View, capacity deployment.Capacity) deployment.Enrollment { t.Helper() nodes := deploymentService(t, s) - token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), deployment.Capacity{MaxActive: 4, MaxRetained: 16})) + token, err := EnrollmentTestToken(nodes.CreateEnrollment(t.Context(), capacity)) if err != nil { t.Fatal(err) } diff --git a/services/core/tests/integration/session_deletion_test.go b/services/core/tests/integration/session_deletion_test.go index b62dc8629..724ad4555 100644 --- a/services/core/tests/integration/session_deletion_test.go +++ b/services/core/tests/integration/session_deletion_test.go @@ -396,7 +396,7 @@ func TestSessionDeletionKeepsProvisioningInputPlacementUntilSettled(t *testing.T var current state if err := s.pool.QueryRow(ctx, `SELECT s.deleted_at, p.released_at FROM sessions s JOIN environments e ON e.session_id=s.id JOIN runtime_placements p ON p.environment_id=e.id - WHERE s.id=$1 AND p.node_id=$2`, session.ID, d.LocalNodeID).Scan(¤t.deleted, ¤t.released); err != nil { + WHERE s.id=$1 AND p.node_id=$2`, session.ID, d.NodeID).Scan(¤t.deleted, ¤t.released); err != nil { t.Fatal("missing placement", err) } nodes, err := deploymentService(t, s).ListNodes(ctx) diff --git a/services/core/tests/integration/web_deployment_only_migration_test.go b/services/core/tests/integration/web_deployment_only_migration_test.go new file mode 100644 index 000000000..1bbec6ad9 --- /dev/null +++ b/services/core/tests/integration/web_deployment_only_migration_test.go @@ -0,0 +1,39 @@ +package integration + +import ( + "strings" + "testing" + + "github.com/google/uuid" +) + +// A process-configured deployment refuses the upgrade; a Web-managed one +// keeps its installation and reset in both directions. +func TestWebDeploymentOnlyMigrationRefusesProcessDeployment(t *testing.T) { + db, provider := runtimeNamesMigrationSchema(t) + ctx := t.Context() + if _, err := provider.UpTo(ctx, 92); err != nil { + t.Fatal(err) + } + installation := uuid.NewString() + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET installation_id=$1, backend_fingerprint=$2`, installation, strings.Repeat("a", 64)); err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 93); err == nil || !strings.Contains(err.Error(), "configured outside Web setup") { + t.Fatal("process deployment upgraded", err) + } + if _, err := db.ExecContext(ctx, `UPDATE runtime_deployment SET web_managed=true, provider_kind='docker', mode='nodes', generation=1, + admission_paused=true, reset_clear='force', reset_requested_at=now(), reset_forced_at=now(), reset_audit='{}'`); err != nil { + t.Fatal(err) + } + if _, err := provider.UpTo(ctx, 93); err != nil { + t.Fatal(err) + } + if _, err := provider.DownTo(ctx, 92); err != nil { + t.Fatal(err) + } + var restored bool + if err := db.QueryRowContext(ctx, `SELECT web_managed AND local_node_id IS NULL AND admission_paused AND installation_id=$1 FROM runtime_deployment`, installation).Scan(&restored); err != nil || !restored { + t.Fatal("downgrade lost the Web installation", err) + } +} diff --git a/services/core/tests/integration/worker_fixture_test.go b/services/core/tests/integration/worker_fixture_test.go index 872c77345..790749499 100644 --- a/services/core/tests/integration/worker_fixture_test.go +++ b/services/core/tests/integration/worker_fixture_test.go @@ -4,11 +4,17 @@ import ( "context" "errors" "testing" + "time" + "github.com/google/uuid" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/modelconfigurationpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/sessionpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) @@ -110,3 +116,76 @@ func fixtureOwner(s *Store, lease *pgunit.Lease) (execution.Owner, error) { Sessions: sessionExecution, }, nil } + +// webDeployment claims s's deployment for a new installation and selects +// provider on it as Web setup does, then closes its execution lease so a +// Worker can start. It returns the installation. +func webDeployment(t *testing.T, s *Store, provider string) string { + t.Helper() + lease, err := pgunit.AcquireLease(t.Context(), s.pool) + if err != nil { + t.Fatal(err) + } + defer lease.Close(context.Background()) + owner, err := fixtureOwner(s, lease) + if err != nil { + t.Fatal(err) + } + installation := uuid.NewString() + input := sandbox.Selection{Provider: provider, DeploymentSpec: SandboxDeploymentTestSpec(provider)} + if provider == "e2b" { + input = e2bSelection() + } + if err := owner.Deployment.Claim(t.Context(), installation); err != nil { + t.Fatal(err) + } + if _, err := owner.Deployment.Initialize(t.Context(), installation, input); err != nil { + t.Fatal(err) + } + return installation +} + +// webRuntimes is the Worker's sandbox runtimes as cmd/server builds them for +// installation: a deferred provider that runs s's committed Web setup on p. +func webRuntimes(t testing.TB, s *Store, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.RuntimeProvider { + deployments := deploymentService(t, s) + return execution.NewDeferredRuntimeProvider(installation, func(ctx context.Context) (*execution.RuntimeProvider, error) { + setup, err := deployments.Setup(ctx) + if err != nil || setup.Provider == "" { + return nil, err + } + return &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Mode: setup.Mode, Generation: setup.Generation, + CoreURL: "http://core.invalid/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: p, Suspension: suspension}, nil + }, unusedPreparation(t)) +} + +// unusedPreparation is the preparer of a test that submits no sandbox +// selection through the Worker; preparing one fails the test. +func unusedPreparation(t testing.TB) execution.RuntimeDeploymentPreparer { + return func(context.Context, deployment.Setup) (execution.PreparedRuntimeDeployment, error) { + t.Error("the test prepared a sandbox selection it did not submit") + return execution.PreparedRuntimeDeployment{}, errors.New("unexpected sandbox selection preparation") + } +} + +// startWebWorker starts the Worker on webRuntimes. +func startWebWorker(t *testing.T, s *Store, registry *runtimegateway.Registry, installation string, p sandbox.SandboxProvider, suspension *execution.RuntimeSuspensionPolicy) *execution.Worker { + t.Helper() + w, err := startNextWorker(t.Context(), s, &execution.Dispatcher{Registry: registry, ManagedRuntimes: webRuntimes(t, s, installation, p, suspension)}) + if err != nil { + t.Fatal(err) + } + return w +} + +// startNextWorker is startWorkerErr after another owner closed its lease. A +// closed lease stays held until PostgreSQL ends its backend, so startup +// retries ErrLeaseHeld briefly. +func startNextWorker(ctx context.Context, s *Store, dispatcher *execution.Dispatcher) (*execution.Worker, error) { + for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(20 * time.Millisecond) { + w, err := startWorkerErr(ctx, s, dispatcher) + if !errors.Is(err, pgunit.ErrLeaseHeld) || time.Now().After(deadline) { + return w, err + } + } +}