From 01ac5d10ef2c001df135a1b4fdcb97951a85b010 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 7 Oct 2026 17:25:02 +0000 Subject: [PATCH] Load process settings once --- AGENTS.md | 2 +- contracts/agents-api/admin-api.md | 2 +- .../environment-executor-credentials.md | 2 +- contracts/agents-api/environments.md | 2 +- contracts/agents-api/model-execution.md | 2 +- contracts/agents-api/runtime-observability.md | 2 +- contracts/agents-api/vaults.md | 2 +- contracts/agents-api/zh/admin-api.md | 4 +- .../zh/environment-executor-credentials.md | 4 +- contracts/agents-api/zh/environments.md | 4 +- contracts/agents-api/zh/model-execution.md | 4 +- .../agents-api/zh/runtime-observability.md | 4 +- contracts/agents-api/zh/vaults.md | 4 +- deploy/compose/compose.yaml | 5 +- deploy/compose/test_compose.py | 2 +- deploy/distribution/Runtime.Dockerfile | 3 +- deploy/install.dev.sh | 2 +- docs/configuration.md | 65 +-- docs/web/console-server.md | 6 +- docs/zh/configuration.md | 67 +-- docs/zh/web/console-server.md | 8 +- internal/obs/log/init.go | 57 ++- internal/obs/log/init_test.go | 30 ++ services/core/IMPLEMENTATION.md | 2 +- services/core/cmd/oac/main.go | 5 +- services/core/cmd/server/core_metrics.go | 26 +- services/core/cmd/server/credential_cipher.go | 26 -- .../core/cmd/server/credential_cipher_test.go | 53 --- services/core/cmd/server/daemon_bootstrap.go | 25 - services/core/cmd/server/installation.go | 25 +- services/core/cmd/server/main.go | 188 +++----- services/core/cmd/server/managed_nodes.go | 72 +-- services/core/cmd/server/managed_setup.go | 7 +- .../core/cmd/server/managed_setup_test.go | 30 +- services/core/cmd/server/oauth_refresh.go | 18 - .../core/cmd/server/oauth_refresh_test.go | 18 - .../core/cmd/server/process_configuration.go | 23 - .../cmd/server/process_configuration_test.go | 50 -- services/core/cmd/server/runtime_history.go | 136 +----- .../core/cmd/server/runtime_history_test.go | 68 +-- services/core/cmd/server/write_audit.go | 49 -- services/core/cmd/server/write_audit_test.go | 51 --- services/core/deploy/claude/Dockerfile | 1 - .../core/internal/api/contract_routes_test.go | 2 +- services/core/internal/api/dependencies.go | 4 +- .../core/internal/api/dependencies_test.go | 2 +- .../internal/api/environment_installation.go | 6 +- .../api/environment_installation_test.go | 2 +- .../api/project_api_key_configuration.go | 3 - .../internal/api/project_api_keys_test.go | 3 - services/core/internal/coremetrics/service.go | 85 +++- .../core/internal/coremetrics/service_test.go | 55 ++- .../core/internal/deployment/public_url.go | 43 +- .../core/internal/deployment/rules_test.go | 12 +- .../internal/environmentconfig/setup_test.go | 2 +- .../bootstrap_interrupt_unix_test.go | 2 +- .../core/internal/nativeinstaller/catalog.go | 9 +- .../internal/nativeinstaller/catalog_test.go | 6 + .../core/internal/processconfig/config.go | 433 ++++++++++++------ .../internal/processconfig/config_test.go | 238 ++++++++-- services/core/internal/runtime/gateway.go | 6 +- services/core/internal/runtimeobs/sampler.go | 49 +- .../core/internal/runtimeobs/sampler_test.go | 80 +--- services/web/Dockerfile | 1 - services/web/config.go | 29 +- services/web/config_test.go | 15 +- services/web/main.go | 27 +- 67 files changed, 1053 insertions(+), 1217 deletions(-) create mode 100644 internal/obs/log/init_test.go delete mode 100644 services/core/cmd/server/credential_cipher.go delete mode 100644 services/core/cmd/server/credential_cipher_test.go delete mode 100644 services/core/cmd/server/daemon_bootstrap.go delete mode 100644 services/core/cmd/server/oauth_refresh.go delete mode 100644 services/core/cmd/server/oauth_refresh_test.go delete mode 100644 services/core/cmd/server/process_configuration.go delete mode 100644 services/core/cmd/server/process_configuration_test.go delete mode 100644 services/core/cmd/server/write_audit.go delete mode 100644 services/core/cmd/server/write_audit_test.go diff --git a/AGENTS.md b/AGENTS.md index d6e200722..364d6fb72 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Do not multiply entities without necessity. The long-term goal is minimal code, Each setting and each piece of data is written in one place and read from that place: no second copy, no environment-variable or file fallback and no alias. A new setting joins its category and lives beside its peers. -The categories are [process settings](docs/configuration.md#process-settings-configjson), [derived files](docs/configuration.md#how-oac-apply-works), [secrets](docs/configuration.md#installation-directory), and Core's database for [runtime settings](docs/configuration.md#runtime-settings-web) and execution data. [Configuration](docs/configuration.md) owns the installation layout and the settings themselves. +The categories are [process settings](docs/configuration.md#process-settings), [derived files](docs/configuration.md#how-oac-apply-works), [secrets](docs/configuration.md#installation-directory), and Core's database for [runtime settings](docs/configuration.md#runtime-settings-web) and execution data. [Configuration](docs/configuration.md) owns the installation layout and the settings themselves. ### Pre-release: no compatibility layers diff --git a/contracts/agents-api/admin-api.md b/contracts/agents-api/admin-api.md index 81d361121..ca0356b5c 100644 --- a/contracts/agents-api/admin-api.md +++ b/contracts/agents-api/admin-api.md @@ -132,7 +132,7 @@ Core writes this record in the same transaction that creates the Session. Later | Field | Meaning | | --- | --- | | `object` | `core.installation` | -| `installation_id` | The installation ID from `state.json` ([installation directory](../../docs/configuration.md#installation-directory)); null when Core runs without the sandbox manager | +| `installation_id` | The installation ID from `OAC_INSTALLATION_ID_FILE` ([Compose installations](../../docs/configuration.md#compose-installations)); null when Core runs without the sandbox manager | | `public_url` | The [`public_url`](../../docs/configuration.md#settings) setting: the origin applications, nodes, sandboxes and self-hosted executors use. Null when unset | | `api_base_url` | `public_url` followed by `/v1`, the `OPENAI_BASE_URL` for Project API keys. Null when `public_url` is null | | `local_only` | True when `public_url` names a loopback host, which only the Core host reaches | diff --git a/contracts/agents-api/environment-executor-credentials.md b/contracts/agents-api/environment-executor-credentials.md index fad59755c..bd58ce767 100644 --- a/contracts/agents-api/environment-executor-credentials.md +++ b/contracts/agents-api/environment-executor-credentials.md @@ -35,7 +35,7 @@ The installer calls these machine routes on Core: | `POST /api/v1/agent-daemon/installation` | Grant | The frozen binding: `version`, `protocol_version`, `environment_id`, `remote_url`, `workspace_directory`, `harness` | | `POST /api/v1/agent-daemon/installation/claim` | Grant | `{"executor_token":"SECRET"}`; 204 | -An invalid or expired grant returns 401 `installation_authorization_invalid`. Without matching installers the grant routes return 503 `installation_unavailable`. Core signs each grant with the installation's [`secrets/credential.key`](../../docs/configuration.md#installation-directory); without a configured key, the Session responses and Core-key read above and the grant routes return 503 `credential_storage_unavailable`. A malformed secret returns 400. Artifact routes carry no credential, and the grant is sent only to Core, never to an artifact host. +An invalid or expired grant returns 401 `installation_authorization_invalid`. Without matching installers the grant routes return 503 `installation_unavailable`. Core signs each grant with the installation's [`secrets/core/credential.key`](../../docs/configuration.md#compose-installations); without a configured key, the Session responses and Core-key read above and the grant routes return 503 `credential_storage_unavailable`. A malformed secret returns 400. Artifact routes carry no credential, and the grant is sent only to Core, never to an artifact host. ## Core-key routes diff --git a/contracts/agents-api/environments.md b/contracts/agents-api/environments.md index b7258d79d..4ac2eece0 100644 --- a/contracts/agents-api/environments.md +++ b/contracts/agents-api/environments.md @@ -271,7 +271,7 @@ session = client.beta.agents.sessions.create( - Responses carry safe metadata and never `env`, `setup_commands` bodies or inline file data. - List uses `after`, `limit` (default 20; 0 is treated as 1 and values above 100 as 100) and `order` (default `desc`), ordered by creation time and ID. Missing and foreign Template IDs and cursors return the same 404. - Update: an omitted field keeps its value and a supplied field replaces it. Null clears `name` and every list and resets `network` to enabled. -- Writes and Session resolution that seal or open confidential content (files, env, setup commands, Skills, Plugins) need Core's [credential key](../../docs/configuration.md#installation-directory); metadata reads do not. +- Writes and Session resolution that seal or open confidential content (files, env, setup commands, Skills, Plugins) need Core's [credential key](../../docs/configuration.md#compose-installations); metadata reads do not. - A Session resolves `environment_template_id` within its Project once, at creation, freezes the effective configuration and never passes the Template ID to the Provider or Runtime. Updating or deleting a Template never changes an existing Session. Creation retries recover the recorded caller intent before reading the Template, even after it is deleted; a changed intent conflicts. ### Inheritance diff --git a/contracts/agents-api/model-execution.md b/contracts/agents-api/model-execution.md index 38bc231b0..8d8751577 100644 --- a/contracts/agents-api/model-execution.md +++ b/contracts/agents-api/model-execution.md @@ -57,7 +57,7 @@ The deployment default holds the operator's key, so it stays on operator compute An empty Session execution extension is invalid. An explicit null provider requests inheritance; an empty or partial provider object is invalid. Unknown, duplicate or output-only saved-provider fields are rejected. A saved Agent without a Harness may save a valid bundle; its Harness compatibility is checked at Session admission. A provider-only Agent update keeps the saved Harness and validates the merged combination under the row lock. The Session's inline `agent.x_agents_core` accepts `harness` and `harness_config`; the provider override belongs at the request's top level. -Core reads the Agent configuration and encrypted bundle from one database snapshot; an explicit complete Session override needs no decryption of the saved bundle. The Session's own encrypted snapshot is written atomically with the Session and its Environment. Existing Sessions never consult the Agent again: edits, key replacement, deletion, suspension and restarts cannot change their model, Harness or provider. A missing or wrong encryption key fails closed; keep the same [credential key](../../docs/configuration.md#installation-directory) across restarts. There is no Turn-level override. +Core reads the Agent configuration and encrypted bundle from one database snapshot; an explicit complete Session override needs no decryption of the saved bundle. The Session's own encrypted snapshot is written atomically with the Session and its Environment. Existing Sessions never consult the Agent again: edits, key replacement, deletion, suspension and restarts cannot change their model, Harness or provider. A missing or wrong encryption key fails closed; keep the same [credential key](../../docs/configuration.md#compose-installations) across restarts. There is no Turn-level override. New hosted requests, and requests that omit the inline model, record caller intent before resolving mutable defaults. Other inline requests, such as `none`, keep the resolved-request retry rule; that hash leaves out the deployment default, so changing the default does not change their retry identity. A matching creation retry recovers the committed Session before resolving the Agent or provider again and enqueues no further input. Streaming is outside the retry identity. The [TypeScript client](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/packages/agents-client/README.md#saved-agent-and-deployment-defaults) shows saved Agents and deployment defaults. diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index 226fc2ca8..31fe8e4ff 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -89,7 +89,7 @@ Periodic collection runs only with the execution worker (Core started with `OAC_ The sampler sweeps once at startup and again each sampling interval after the previous sweep ends. A sweep is a keyset scan, in Session ID order, of the Sessions that are not deleted, are `openai_hosted` and have no released allocation. It reads pages of 32 Sessions through the same resolver and sources as current reads, with eight concurrent reads and two seconds per source. The sampler checks the lease before each page and every 100 ms during a sweep, cancels in-flight reads when ownership is lost, and checks it again before handing each record to export. A failed row does not stop the sweep, and an incomplete sweep is repeated at the next interval. -Every observation, current or periodic, is marked with its collection source, `on_read` or `periodic`, and handed to each exporter's bounded queue. A full queue drops the record, which becomes a missing sample, never a zero. The PostgreSQL history store and the optional OTLP exporter have independent queues, so an exporter outage cannot delay local history or execution. The [`core.runtime_history` settings](../../docs/configuration.md#settings) set the interval, queue capacity, timeout and OTLP destination. +Every observation, current or periodic, is marked with its collection source, `on_read` or `periodic`, and handed to each exporter's bounded queue. A full queue drops the record, which becomes a missing sample, never a zero. The PostgreSQL history store and the optional OTLP exporter have independent queues, so an exporter outage cannot delay local history or execution. The [Runtime history file](../../docs/configuration.md#runtime-history-file) sets the interval, queue capacity, timeout and OTLP destination. ### Stored history diff --git a/contracts/agents-api/vaults.md b/contracts/agents-api/vaults.md index 938ae2f43..a05e76511 100644 --- a/contracts/agents-api/vaults.md +++ b/contracts/agents-api/vaults.md @@ -137,7 +137,7 @@ Token endpoints must be HTTPS. Core resolves the host, rejects loopback, private ## Storage key -Core seals every token, refresh token and client secret with AES-256-GCM under the installation's [`secrets/credential.key`](../../docs/configuration.md#installation-directory), bound to the Project, Vault, Credential, auth type and `mcp_server_url`. A wrong key, a modified row or a row moved to another binding fails to decrypt. Names are metadata outside the binding. The key and plaintext tokens exist in trusted service memory; encryption protects stored secrets and does not protect against a compromised service host. +Core seals every token, refresh token and client secret with AES-256-GCM under the installation's [`secrets/core/credential.key`](../../docs/configuration.md#compose-installations), bound to the Project, Vault, Credential, auth type and `mcp_server_url`. A wrong key, a modified row or a row moved to another binding fails to decrypt. Names are metadata outside the binding. The key and plaintext tokens exist in trusted service memory; encryption protects stored secrets and does not protect against a compromised service host. Without a configured key, Credential creation and replacement return 503 `credential_storage_unavailable` before writing; reads, lists, deletion and Vault operations still work. An unreadable or malformed key file stops Core at startup. Losing or replacing the key makes every stored secret unusable; Core supports one key, with no rotation or re-encryption. diff --git a/contracts/agents-api/zh/admin-api.md b/contracts/agents-api/zh/admin-api.md index 389c07997..8ce891d43 100644 --- a/contracts/agents-api/zh/admin-api.md +++ b/contracts/agents-api/zh/admin-api.md @@ -1,7 +1,7 @@ --- title: "Core 管理 API" source: contracts/agents-api/admin-api.md -source_hash: 7759541dbc59dab917499f506c9e9c11184e8065fd1ed6fb418baaf4a478faf3 +source_hash: 7eb295db6402db8dae91fcdd97f90a5900f740925caaee9d0172b9886544f6ec --- Core 管理 API(`/core/v1`)用于管理安装实例:Project 及其 API 密钥、Project 资源的读取和删除、执行器凭据、部署默认模型、沙箱部署及其节点、监控和审计。Web 的[控制台服务器](../../../docs/zh/web/console-server.md#forwarding-to-core)会为已登录的管理员调用它;运维人员则从 Core 主机上的脚本调用它([编写 Core API 脚本](../../../docs/zh/getting-started/operations.md#script-the-core-api))。生成的架构是 [core.openapi.yaml](../core.openapi.yaml),所有错误都使用 [Core 错误封装](core-errors.md)。 @@ -134,7 +134,7 @@ Core 会在创建 Session 的同一事务中写入此记录。之后的 Agent | 字段 | 含义 | | --- | --- | | `object` | `core.installation` | -| `installation_id` | `state.json` 中的安装 ID([安装目录](../../../docs/zh/configuration.md#installation-directory));Core 在不使用沙箱管理器运行时为 null | +| `installation_id` | `OAC_INSTALLATION_ID_FILE` 中的安装 ID([Compose 安装](../../../docs/zh/configuration.md#compose-installations));Core 在不使用沙箱管理器运行时为 null | | `public_url` | `public_url` 设置([设置](../../../docs/zh/configuration.md#settings)):应用程序、节点、沙箱和自托管执行器使用的源地址。未设置时为 null | | `api_base_url` | 在 `public_url` 后附加 `/v1`,即 Project API 密钥使用的 `OPENAI_BASE_URL`。当 `public_url` 为 null 时为 null | | `local_only` | 当 `public_url` 指向回环主机时为 True,该主机只能由 Core 主机访问 | diff --git a/contracts/agents-api/zh/environment-executor-credentials.md b/contracts/agents-api/zh/environment-executor-credentials.md index d78e8c421..3d1fea6bf 100644 --- a/contracts/agents-api/zh/environment-executor-credentials.md +++ b/contracts/agents-api/zh/environment-executor-credentials.md @@ -1,7 +1,7 @@ --- title: "Environment 执行器凭证" source: contracts/agents-api/environment-executor-credentials.md -source_hash: 6c1db305481f9ab2a51bdd0c88243feaab48348b71f5f3ebbc8f00b17744f412 +source_hash: 725257a92518431a4b942ea35187e37bb171f890d7797e843a9f4eb62f47ad4b --- 执行器凭证允许 `oac-daemon` 为一个 `self_hosted` Environment 注册并连接。它只授权该 Environment 的私有 daemon 传输(`/api/v1/agent-daemon/*`),不授权 `/v1`、`/core/v1`、sandbox node 注册或 Project 资源。Project 的 principal 是其执行 principal。Core 只保存密钥摘要。 @@ -37,7 +37,7 @@ grant 绑定 Environment、Session 创建者的 principal 和 Core 构建版本 | `POST /api/v1/agent-daemon/installation` | Grant | 固定绑定:`version`、`protocol_version`、`environment_id`、`remote_url`、`workspace_directory`、`harness` | | `POST /api/v1/agent-daemon/installation/claim` | Grant | `{"executor_token":"SECRET"}`;204 | -无效或过期的 grant 返回 401 `installation_authorization_invalid`。没有匹配安装器时,grant 路由返回 503 `installation_unavailable`。Core 用安装的 [`secrets/credential.key`](../../../docs/zh/configuration.md#installation-directory) 签名每个 grant;未配置 key 时,上述 Session 响应、Core-key 查询和 grant 路由返回 503 `credential_storage_unavailable`。格式错误的密钥返回 400。产物路由不携带凭证,grant 只发送给 Core,不发送给产物主机。 +无效或过期的 grant 返回 401 `installation_authorization_invalid`。没有匹配安装器时,grant 路由返回 503 `installation_unavailable`。Core 用安装的 [`secrets/core/credential.key`](../../../docs/zh/configuration.md#compose-installations) 签名每个 grant;未配置 key 时,上述 Session 响应、Core-key 查询和 grant 路由返回 503 `credential_storage_unavailable`。格式错误的密钥返回 400。产物路由不携带凭证,grant 只发送给 Core,不发送给产物主机。 ## Core-key 路由 {#core-key-routes} diff --git a/contracts/agents-api/zh/environments.md b/contracts/agents-api/zh/environments.md index 221d34c20..1891bd481 100644 --- a/contracts/agents-api/zh/environments.md +++ b/contracts/agents-api/zh/environments.md @@ -1,7 +1,7 @@ --- title: "环境与模板" source: contracts/agents-api/environments.md -source_hash: a93a477f3de39b2206702995821282404c29ee997ef6b782180d4972bada43f7 +source_hash: 8fb6cbd0b8daed4686d1b6829a49e799f03bb78c6bdb1f93cdb9a4518fec4f8f --- Environment 是 Session 的执行资源,包括 Harness 运行所在的机器、工作区以及已完成准备的能力。Session 通过其 `environment` 配置创建 Environment;不存在独立的 create 调用。Environment Template 是 Session 创建时解析的可复用准备配置。本契约涵盖这两类资源、两种放置方式、输入接纳、能力准备、Skills、Plugins 和 MCP 连接来源。 @@ -273,7 +273,7 @@ session = client.beta.agents.sessions.create( - 响应会携带安全元数据,绝不会包含 `env`、`setup_commands` 正文或内联文件数据。 - 列表操作使用 `after`、`limit`(默认 20;0 按 1 处理,超过 100 的值按 100 处理)和 `order`(默认 `desc`),并按创建时间和 ID 排序。不存在和属于外部 Project 的 Template ID 及游标都会返回相同的 404。 - 更新时,省略字段会保留原值,提供字段则会替换原值。Null 会清除 `name` 和每个列表,并将 `network` 重置为启用。 -- 封装或解封机密内容(文件、env、设置命令、Skills、Plugins)的写入操作和 Session 解析需要 Core 的 [credential key](../../../docs/zh/configuration.md#installation-directory);元数据读取则不需要。 +- 封装或解封机密内容(文件、env、设置命令、Skills、Plugins)的写入操作和 Session 解析需要 Core 的 [credential key](../../../docs/zh/configuration.md#compose-installations);元数据读取则不需要。 - Session 会在创建时于其 Project 内解析一次 `environment_template_id`,冻结有效配置,并且绝不将 Template ID 传递给 Provider 或 Runtime。更新或删除 Template 绝不会改变现有 Session。创建重试会在读取 Template 之前恢复已记录的调用方意图,即使 Template 已删除也是如此;意图发生变化时会产生冲突。 ### 继承 {#inheritance} diff --git a/contracts/agents-api/zh/model-execution.md b/contracts/agents-api/zh/model-execution.md index aa762e519..ca4231c01 100644 --- a/contracts/agents-api/zh/model-execution.md +++ b/contracts/agents-api/zh/model-execution.md @@ -1,7 +1,7 @@ --- title: "模型执行" source: contracts/agents-api/model-execution.md -source_hash: b995996e38d7a1591d1db0a548a616f6c0ac687f36185a13e95cb52d2e2ff0c3 +source_hash: ee651cc7bb506eab33a819aa40a97095270574a793dea95784104f19692fa4ec --- 每个 Session 都运行一个 Harness,并使用一个模型提供商。Core 通过三个固定版本上游协议未定义的 Core 扩展来选择它们:`x_agents_core.harness` 选择 Harness,`x_agents_core.model_provider` 提供端点和密钥,`x_agents_core.harness_config` 携带原生模型参数。Core 没有提供商目录、模型别名解析或产品权限模型;除 Session 和已保存 Agent 配置包外,唯一存储的配置包是每个 Harness 的一个 [deployment default](#deployment-defaults)。本文档定义 Harness—模型提供商协议:[`internal/modelprovider/config.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/modelprovider/config.go) 负责验证冻结的提供商连接,每个 Harness 则通过 [`internal/harnessconfig/harness.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/internal/harnessconfig/harness.go) 声明其协议和原生参数。 @@ -59,7 +59,7 @@ MiniMax Code 要求上下文限制和输出限制均为正数。Core 会在写 空的 Session 执行扩展无效。显式 null 提供商会请求继承;空的或不完整的提供商对象无效。已保存提供商配置中的未知字段、重复字段或只读输出字段均会被拒绝。没有 Harness 的已保存 Agent 可以保存有效配置包;其 Harness 兼容性会在 Session 准入时检查。仅更新提供商的 Agent 更新会保留已保存的 Harness,并在行锁保护下验证合并后的组合。Session 内联的 `agent.x_agents_core` 接受 `harness` 和 `harness_config`;提供商覆盖值必须放在请求顶层。 -Core 从同一个数据库快照读取 Agent 配置和加密配置包;显式提供完整 Session 覆盖值时,无需解密已保存的配置包。Session 自身的加密快照会与 Session 及其 Environment 原子写入。现有 Session 绝不会再次查询 Agent:Agent 编辑、密钥替换、删除、暂停和重启均无法改变其模型、Harness 或提供商。加密密钥缺失或错误时会安全失败;重启前后应保持相同的 [credential key](../../../docs/zh/configuration.md#installation-directory)。不存在 Turn 级覆盖。 +Core 从同一个数据库快照读取 Agent 配置和加密配置包;显式提供完整 Session 覆盖值时,无需解密已保存的配置包。Session 自身的加密快照会与 Session 及其 Environment 原子写入。现有 Session 绝不会再次查询 Agent:Agent 编辑、密钥替换、删除、暂停和重启均无法改变其模型、Harness 或提供商。加密密钥缺失或错误时会安全失败;重启前后应保持相同的 [credential key](../../../docs/zh/configuration.md#compose-installations)。不存在 Turn 级覆盖。 新的托管请求以及省略内联模型的请求,会在解析可变默认值之前记录调用方意图。其他内联请求,例如 `none`,继续遵循已解析请求的重试规则;该哈希不包含部署默认值,因此更改默认值不会改变其重试标识。匹配的创建重试会在再次解析 Agent 或提供商之前恢复已提交的 Session,并且不会进一步加入输入。流式传输不参与重试标识的计算。[TypeScript client](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/packages/agents-client/README.md#saved-agent-and-deployment-defaults) 展示了已保存 Agent 和部署默认值。 diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index 8e24b9a56..ddd10ea91 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -1,7 +1,7 @@ --- title: "运行时可观测性" source: contracts/agents-api/runtime-observability.md -source_hash: f79a077350118f5f9bb3f4e8874242af3cb716e92f5e090f6652b001ae5780a1 +source_hash: 103575f3971e77d5ba149cacb27e972e429a46713b2bda7da36cae43bbf313fa --- 这是面向贡献者的契约,规定 Core 如何观测 Runtime 并保留其历史。路由和响应字段见 [Runtime telemetry API](runtime-observability-api.md)。代码位于 `services/core/internal/runtimeobs`(解析、源、采样器和导出)、`internal/runtimehistory`(历史查询和 PostgreSQL 存储)以及 `internal/runtimeobs/otlpexporter`。 @@ -91,7 +91,7 @@ CPU 静默状态、心跳时龄、连接状态和保活时间都不是空闲时 采样器在启动时扫描一次,此后每次扫描结束后再经过一个采样间隔再次扫描。一次扫描按 Session ID 顺序,对未删除、状态为 `openai_hosted` 且没有已释放分配的 Session 执行 keyset 扫描。它通过与当前读取相同的解析器和源,以 32 个 Session 为一页进行读取,并发数为 8,每个源时限为 2 秒。采样器在每页之前以及扫描期间每 100 ms 检查租约;失去所有权时取消进行中的读取;将每条记录交给导出之前再次检查租约。失败的行不会停止扫描;未完成的扫描会在下一个间隔重复。 -每次观测,无论来自当前读取还是周期采集,都会标记采集源 `on_read` 或 `periodic`,并放入每个导出器的有界队列。队列已满时会丢弃记录;该记录将成为缺失采样,而绝不会成为零。PostgreSQL 历史存储和可选 OTLP 导出器使用彼此独立的队列,因此导出器故障不会延迟本地历史记录或执行。[`core.runtime_history` settings](../../../docs/zh/configuration.md#settings) 用于设置采样间隔、队列容量、超时和 OTLP 目标。 +每次观测,无论来自当前读取还是周期采集,都会标记采集源 `on_read` 或 `periodic`,并放入每个导出器的有界队列。队列已满时会丢弃记录;该记录将成为缺失采样,而绝不会成为零。PostgreSQL 历史存储和可选 OTLP 导出器使用彼此独立的队列,因此导出器故障不会延迟本地历史记录或执行。[Runtime 历史文件](../../../docs/zh/configuration.md#runtime-history-file) 用于设置采样间隔、队列容量、超时和 OTLP 目标。 ### 存储的历史记录 {#stored-history} diff --git a/contracts/agents-api/zh/vaults.md b/contracts/agents-api/zh/vaults.md index 0f5dd927d..81434de67 100644 --- a/contracts/agents-api/zh/vaults.md +++ b/contracts/agents-api/zh/vaults.md @@ -1,7 +1,7 @@ --- title: "Vault 与 Credential" source: contracts/agents-api/vaults.md -source_hash: 9e56ee84c782d1f9c0f5506f960a275d9afa3e554634131bf09a74e736135926 +source_hash: 95626340ee36faee3418dd1155a0e50c378ead09c09c91e86f30e09bd8f409e0 --- Vault 是 Project 所有的 Credential 容器。Credential 保存一个 HTTPS MCP server 的秘密:`static_bearer` token 或 `mcp_oauth` grant。Session 在 `vault_ids` 中关联 Vault;Core 在创建 Session 时为每个 HTTP MCP server 选择一个 Credential,只在分派工作时将解密 token 交给 Runtime。秘密只能写入:任何读取都不返回 token、refresh token、client secret 或密文。 @@ -139,7 +139,7 @@ Token endpoint 必须为 HTTPS。Core 解析主机,拒绝回环、私有、链 ## 存储密钥 {#storage-key} -Core 使用安装的 [`secrets/credential.key`](../../../docs/zh/configuration.md#installation-directory),以 AES-256-GCM 加密每个 token、refresh token 和 client secret,绑定 Project、Vault、Credential、auth type 和 `mcp_server_url`。错误密钥、修改的行或移动到其他绑定的行均无法解密。名称是不参与绑定的元数据。key 和明文 token 存在于可信服务内存中;加密保护存储的秘密,不保护已被攻破的服务主机。 +Core 使用安装的 [`secrets/core/credential.key`](../../../docs/zh/configuration.md#compose-installations),以 AES-256-GCM 加密每个 token、refresh token 和 client secret,绑定 Project、Vault、Credential、auth type 和 `mcp_server_url`。错误密钥、修改的行或移动到其他绑定的行均无法解密。名称是不参与绑定的元数据。key 和明文 token 存在于可信服务内存中;加密保护存储的秘密,不保护已被攻破的服务主机。 未配置 key 时,Credential 创建和替换在写入前返回 503 `credential_storage_unavailable`;读取、列表、删除和 Vault 操作仍可用。key 文件不可读或格式错误会使 Core 启动失败。丢失或替换 key 使全部已存储秘密无法使用;Core 只支持一个 key,不支持轮换或重新加密。 diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 6fe45a4db..70471dc31 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -58,8 +58,6 @@ services: OAC_DATABASE_PASSWORD_FILE: /run/database/password OAC_CREDENTIAL_KEY_FILE: /run/oac/credential.key OAC_CORE_KEY_DIGESTS_FILE: /run/oac/core-key-digests.json - OAC_PROVIDER_STATE_ROOT: /state - OAC_NATIVE_INSTALLER_DIR: /opt/oac/native-installers OAC_EXECUTION_CONCURRENCY: ${OAC_EXECUTION_CONCURRENCY:-} OAC_DEFAULT_HARNESS: ${OAC_DEFAULT_HARNESS:-} OAC_HARNESSES: ${OAC_HARNESSES:-} @@ -99,8 +97,7 @@ services: depends_on: init: {condition: service_completed_successfully} environment: - OAC_WEB_ORIGIN: *public-url - OAC_WEB_UPSTREAM: http://core:8091 + OAC_PUBLIC_URL: *public-url OAC_WEB_CORE_KEY_FILE: /run/oac/core.key OAC_WEB_NODE_PAYLOAD_DIR: /node-payload OAC_LOG_LEVEL: ${OAC_LOG_LEVEL:-} diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index ce428f89a..4f14ef726 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -79,7 +79,7 @@ def test_public_url_can_be_configured_after_initial_startup(self): with self.subTest(public_url=value): configured = self.render(value) expected = value or 'http://localhost:8080' - for name, setting in (('core', 'OAC_PUBLIC_URL'), ('web', 'OAC_WEB_ORIGIN')): + for name, setting in (('core', 'OAC_PUBLIC_URL'), ('web', 'OAC_PUBLIC_URL')): self.assertEqual(configured['services'][name]['environment'][setting], expected) self.assertEqual( {service: [item.get('target') for item in spec.get('volumes', [])] diff --git a/deploy/distribution/Runtime.Dockerfile b/deploy/distribution/Runtime.Dockerfile index 913c1f4a0..886bc08d5 100644 --- a/deploy/distribution/Runtime.Dockerfile +++ b/deploy/distribution/Runtime.Dockerfile @@ -15,8 +15,7 @@ COPY --from=claude /opt/claude-sdk /opt/claude-sdk ENV OAC_RUNTIME_CODEX_BIN=/usr/local/bin/codex \ OAC_RUNTIME_CLAUDE_SDK_NODE=/usr/local/bin/node \ - OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js \ - OAC_RUNTIME_CLAUDE_SDK_WORKSPACE=managed + OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js USER 1000:1000 RUN test "$(codex --version)" = "codex-cli 0.153.4" \ diff --git a/deploy/install.dev.sh b/deploy/install.dev.sh index 7a3fb5135..93579cecf 100755 --- a/deploy/install.dev.sh +++ b/deploy/install.dev.sh @@ -5,7 +5,7 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -install_dir="${OAC_INSTALL_DIR_DEFAULT:-$HOME/.oac/local}" +install_dir="$HOME/.oac/local" host_address="127.0.0.1" web_port="8080" diff --git a/docs/configuration.md b/docs/configuration.md index fcdc5b4a4..5ff249aff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,16 +2,17 @@ title: "Configuration reference" --- -Every setting of a Core installation has exactly one home. There are two kinds: +Every setting of a Core installation has exactly one home, in one of three categories: -| Kind | Examples | Home | Change it with | Takes effect | +| Category | Examples | Home | Change it with | Takes effect | | --- | --- | --- | --- | --- | -| [Process settings](#process-settings-configjson) | Public URL, ports, logging, harnesses, execution concurrency, audit retention, OAuth origins, Runtime history export | `.env` in the installation directory (default `~/.oac/core`) | Edit `.env`, then run `oac apply` | `oac apply` recreates the services that read the changed settings | +| [Process settings](#process-settings) | Public URL, ports, logging, harnesses, execution concurrency, audit retention, OAuth origins, Runtime history | `.env` in the installation directory (default `~/.oac/core`) | Edit `.env`, then run `oac apply` | `oac apply` recreates the services that read the changed settings | +| [Secrets](#compose-installations) | Database password, credential encryption key, installation ID, Core key and the Core key digest derived from it | `secrets/` in the Compose data volume, one copy each | Initialization generates them once; `oac rotate-core-key` replaces the Core key and its digest | `oac rotate-core-key` restarts Core and Web | | [Runtime settings](#runtime-settings-web) | Sandbox backend and size, nodes, Projects and keys, default models, executor credentials | Core's PostgreSQL database | Web, or the Core API (`/core/v1`) with the Core key | Saved without a Core restart; nodes prepare Runtime changes asynchronously | -Web's **System** page shows the installation's addresses, the default models, the sandbox configuration and, under **Startup settings**, the process settings Core loaded. Secrets live in [`secrets/`](#compose-installations), one copy each. No configuration file defines Projects or API keys. +Web's **System** page shows the installation's addresses, the default models, the sandbox configuration and, under **Startup settings**, the process settings Core loaded. No configuration file defines Projects or API keys. -## Process settings {#process-settings-configjson} +## Process settings Installer flags in [installation options](./getting-started/install-options.md) write `.env` once. To change a setting, edit `.env` and apply it: @@ -38,11 +39,11 @@ To change it, point the reverse proxy at the new address first, then edit `OAC_P ### Settings -`OAC_HISTORY_SETTINGS_FILE` may point at a file whose headers hold export credentials. The file stays mode `0600`, and those headers never appear in `oac` output or in the installation report. Model providers are not process settings; see [Default models](#default-models). +Model providers are not process settings; see [Default models](#default-models). | Variable | Default | Meaning | | --- | --- | --- | -| `OAC_PUBLIC_URL` | `http://localhost:8080` | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) | +| `OAC_PUBLIC_URL` | `http://localhost:8080`, set by `compose.yaml`. Core started without it runs no Runtime gateway and executes no Sessions | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) | | `OAC_HOST` | `127.0.0.1` | Web bind address published by `compose.yaml`. The installer sets `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | @@ -53,9 +54,23 @@ To change it, point the reverse proxy at the new address first, then edit `OAC_P | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | | `OAC_OAUTH_TRUSTED_ORIGINS` | unset | Comma-separated HTTPS origins | -| `OAC_HISTORY_SETTINGS_FILE` | unset | Optional Runtime history file. Sensitive; Core reports only whether it is configured | +| `OAC_HISTORY_SETTINGS_FILE` | unset | Optional [Runtime history file](#runtime-history-file). Sensitive; Core reports only whether it is configured | -An unset or empty value selects the default. Edit `.env`, then run `oac apply`. Core reports the process settings it loaded at `GET /core/v1/installation`. `oac-core check-config` validates the same environment without starting Core. Sensitive settings report only whether they are configured. How Core collects and keeps Runtime history is in [retained history](../contracts/agents-api/runtime-observability.md#retained-history-and-optional-export). +An unset or empty value selects the default. Edit `.env`, then run `oac apply`. Core reads every process setting, and every file a setting names, once at startup and reports what it loaded at `GET /core/v1/installation`. `oac-core check-config` loads and validates the same settings and files without starting Core. The native installer catalog under `OAC_PROVIDER_ROOT` is not a setting; Core checks it only when it starts. Errors name the variable, never its value. Sensitive settings report only whether they are configured. + +### Runtime history file + +`OAC_HISTORY_SETTINGS_FILE` names a JSON file that tunes [retained history](../contracts/agents-api/runtime-observability.md#retained-history-and-optional-export) and adds an optional OTLP export. Without it, Core keeps history in its database with the defaults below. In a Compose installation, put the file in the data volume's `secrets/core/` directory, owned by UID 65532 with mode `0600`, and set `OAC_HISTORY_SETTINGS_FILE=/run/oac/`: Core mounts that directory read-only at `/run/oac`. Headers may hold export credentials; they never appear in `oac` output or in the installation report. Unknown fields are rejected. + +| Field | Default | Meaning | +| --- | --- | --- | +| `sample_interval_seconds` | `30` | Periodic sampling interval, from 5 to 300 | +| `queue_capacity` | `256` | Records each exporter queues, at most 4096 | +| `timeout_seconds` | `2` | Export and history query timeout, at most 30 | +| `endpoint` | unset | Absolute OTLP/HTTP metrics URL, such as `https://collector.example/v1/metrics`. Unset, Core exports nothing and the other export fields must be unset | +| `transport` | unset | `otlp_http`; required with `endpoint` | +| `insecure` | `false` | `true` is required for an `http` endpoint and rejected for `https` | +| `headers` | none | Request headers for the endpoint. `Host`, `Content-Length`, `Content-Type` and `Content-Encoding` are reserved | ## Runtime settings: Web @@ -84,7 +99,7 @@ Set a default in **System** → **Default model configuration**, or use `PUT /co ## Compose installations -The [standalone Compose file](./getting-started/install-options.md#docker-compose-and-hosting-platforms) takes process settings from the platform's environment. An empty `OAC_PUBLIC_URL` selects `http://localhost:8080`. Set it to the exact public origin, without a trailing slash, and recreate Core and Web before adding nodes or executors. The platform terminates TLS and routes to `web:8080`. +The [standalone Compose file](./getting-started/install-options.md#docker-compose-and-hosting-platforms) takes process settings from the platform's environment. Set [`OAC_PUBLIC_URL`](#settings) to the exact public origin, without a trailing slash, and recreate Core and Web before adding nodes or executors. The platform terminates TLS and routes to `web:8080`. The initialization service generates secrets and the installation ID once, then verifies them on subsequent deployments. Each secret has one persistent source; Core's key digest is derived from Web's sign-in key. Initialization never replaces missing or changed secrets on an existing installation. Core reads the process environment from `.env`. @@ -94,7 +109,7 @@ The initialization service generates secrets and the installation ID once, then | `secrets/database/` | Generated database password | PostgreSQL and Core | | `secrets/core/` | Credential encryption key, installation ID and Core key digest | Core | | `secrets/web/` | Generated Core sign-in key | Web | -| `state/` | Private Provider state | Core | +| `state/` | Private Provider state, mounted in Core at `/state`. Each adapter owns a subdirectory; E2B uses `e2b/`, with no group or other access | Core | | `node-payload/` | Verified node installation metadata | Web | Initialization prepares this directory; application services receive their secret directories read-only. `docker compose exec web oac-web core-key` prints the Core key to the operator terminal without writing it to container logs. Database passwords and credential encryption keys are never printed. @@ -135,21 +150,17 @@ Core reads its process environment. Compose interpolates `.env` into it and moun | Variable | Set from | | --- | --- | -| `OAC_PUBLIC_URL` | The public origin. Core derives the daemon WebSocket URL, the self-hosted `remote_url`, the hosted sandbox address and the deployment's read-only `core_url` from it, never from request headers. Without it, Core runs no Runtime gateway and executes no Sessions | +| `OAC_PUBLIC_URL` | The [public URL](#settings). Core validates it once and derives the Agents API base, the daemon WebSocket URL, the self-hosted `remote_url`, the installer downloads, the hosted sandbox address and the deployment's read-only `core_url` from it, never from request headers | | `OAC_ADDR` | The image sets `:8091`. Independently started Core defaults to `127.0.0.1:8091` when unset or empty | -| `OAC_DATABASE_URL` | PostgreSQL without a password | +| `OAC_DATABASE_URL` | Required. PostgreSQL without a password | | `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`. The URL must then carry no password | | `OAC_CREDENTIAL_KEY_FILE` | `/run/oac/credential.key` | -| `OAC_CORE_KEY_DIGESTS_FILE` | `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | -| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`: the installation ID, a canonical UUID. It enables the sandbox deployment and node routes and requires `OAC_PUBLIC_URL` and `OAC_CORE_KEY_DIGESTS_FILE`. Core refuses an ID other than the one its database recorded | -| `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS` | The matching [process settings](#settings). `oac-core check-config` validates them without starting Core | -| `OAC_HISTORY_SETTINGS_FILE` | Optional Runtime history file. Sensitive; the installation report says only whether it is set | -| `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | Logging; Web reads the same three | -| `OAC_PROVIDER_ROOT` | Absolute adapter artifact root. The Core image sets `/opt/oac`. Each adapter owns its helper paths beneath this root | -| `OAC_PROVIDER_STATE_ROOT` | Absolute private state root: `/state` in the Core image. Each adapter owns its subdirectory; E2B uses `e2b/`, owned by Core's user with no group or other access. Back it up with the database and `credential.key`; don't mount it into Web or a Runtime | -| `OAC_NATIVE_INSTALLER_DIR` | Self-hosted daemon installers: `/opt/oac/native-installers` in the Compose file. Unset, Core serves none. Core checks the catalog against its own release before serving it | +| `OAC_CORE_KEY_DIGESTS_FILE` | Required. `/run/oac/core-key-digests.json`: a JSON array with the SHA-256 of the Core key | +| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`: the installation ID, a canonical UUID. It enables the sandbox deployment and node routes and requires `OAC_PUBLIC_URL`. Core refuses an ID other than the one its database recorded | +| `OAC_EXECUTION_CONCURRENCY`, `OAC_DEFAULT_HARNESS`, `OAC_HARNESSES`, `OAC_WRITE_AUDIT_RETENTION`, `OAC_OAUTH_TRUSTED_ORIGINS`, `OAC_HISTORY_SETTINGS_FILE`, `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT`, `OAC_LOG_ADD_SOURCE` | The matching [process settings](#settings). Web reads the three log settings too | +| `OAC_PROVIDER_ROOT` | Absolute adapter artifact root. The Core image sets `/opt/oac`. Each adapter owns its helper paths beneath this root. Core serves self-hosted daemon installers from its `native-installers/` directory when that holds a `catalog.json`, after checking the catalog against its own release. Adapter state lives at `/state`, the data volume's [`state/`](#compose-installations) | -Core logs the file paths it loads, never environment values or file contents. +Core logs the history file path it loads, never environment values or file contents. Invalid explicit OAuth trusted origins stop Core at startup. Entries must be HTTPS origins without credentials, query or a non-root path. [Vaults](../contracts/agents-api/vaults.md) owns refresh and network policy. A private issuer also needs a trusted CA: independently managed Unix Core can use Go’s `SSL_CERT_FILE` PEM CA-bundle override, which preserves certificate verification. Managed installation has no custom-CA setting. @@ -159,11 +170,11 @@ Compose sets these for Web. Set them yourself only when you run the console with | Variable | Default | Meaning | | --- | --- | --- | -| `OAC_WEB_ADDR` | `:8080` | Listener address | -| `OAC_WEB_ORIGIN` | `http://127.0.0.1:8080` | The exact browser-facing origin, HTTP or HTTPS, without a path. Host and origin checks use it; HTTPS makes the session cookie `Secure` | -| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core's origin, HTTP or HTTPS, without credentials, query or path | -| `OAC_WEB_CORE_KEY_FILE` | `/admin/core.key` | Absolute path of a regular file with no group or other permissions, holding the Core key: at least 32 characters, no whitespace, at most 4 KiB | +| `OAC_WEB_ADDR` | `:8080` | Listener address. The healthcheck probes it on `127.0.0.1` when its host is empty or unspecified | +| `OAC_PUBLIC_URL` | Required | The [public URL](#settings): the exact browser-facing origin, HTTP or HTTPS, without a path. Host and origin checks use it; HTTPS makes the session cookie `Secure` | +| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core's origin, HTTP or HTTPS, without credentials, query or path. The healthcheck probes its `/healthz` | +| `OAC_WEB_CORE_KEY_FILE` | Required | Absolute path of a regular file with no group or other permissions, holding the Core key: at least 32 characters, no whitespace, at most 4 KiB | | `OAC_WEB_DIST` | `/www` | Absolute directory of the built console; must contain `index.html` | | `OAC_WEB_NODE_PAYLOAD_DIR` | unset | Absolute path of the matched distribution's node payload (the installer's `node-payload/`). Unset, `/node-install/*` is not served and Add node is unavailable | -Defaults apply when a variable is absent; an explicitly empty value is validated as supplied. An invalid `OAC_WEB_*` value stops the console at startup with a message naming the variable. The console also reads `OAC_LOG_LEVEL`, `OAC_LOG_FORMAT` and `OAC_LOG_ADD_SOURCE` ([Core environment](#appendix-core-environment-without-the-installer)); unknown values fall back to their defaults. Use HTTPS for any browser that is not on the same machine. +An unset or empty variable selects its default. An invalid value stops the console at startup with a message naming the variable. The console also reads the three log [process settings](#settings) and rejects the values Core rejects. Use HTTPS for any browser that is not on the same machine. diff --git a/docs/web/console-server.md b/docs/web/console-server.md index 41fa81d3d..52b89daa0 100644 --- a/docs/web/console-server.md +++ b/docs/web/console-server.md @@ -42,7 +42,7 @@ The deployment's reverse proxy sends every path to the console. The console forw Every request except `/healthz`, `/v1`, `/api/v1` and `/docs` must pass these checks first: -1. **Host and origin.** The `Host` header must equal the host of `OAC_WEB_ORIGIN`. An `Origin` header, when present, must equal that origin, and `Sec-Fetch-Site` must be `same-origin` or `none`. A write that carries neither `Origin` nor `Sec-Fetch-Site: same-origin` needs a same-origin `Referer`. Otherwise the console answers 403. `/node-install/*` checks only the host and the path. +1. **Host and origin.** The `Host` header must equal the host of `OAC_PUBLIC_URL`. An `Origin` header, when present, must equal that origin, and `Sec-Fetch-Site` must be `same-origin` or `none`. A write that carries neither `Origin` nor `Sec-Fetch-Site: same-origin` needs a same-origin `Referer`. Otherwise the console answers 403. `/node-install/*` checks only the host and the path. 2. **Safe request.** The path must start with `/` and contain no `%`, backslash, NUL, dot segment or empty segment. Absolute-form request targets, `CONNECT` and `TRACE` get 400. An `Upgrade` header gets 400 except on `/v1`, `/api/v1` and `/docs`, which are forwarded before these checks. A `/core/v1` request can therefore never leave that prefix. 3. **Sign-in.** Paths that need sign-in answer 401 without a valid session cookie. @@ -75,7 +75,7 @@ The console never retries a request. Browser code calls `/core/v1` through the t The administrator signs in with the deployment's [Core key](../getting-started/operations.md#core-key). There are no console accounts, usernames or setup step, and signing in grants the whole console. - The console compares SHA-256 digests of the submitted and configured keys in constant time. It never logs or returns the key. -- The session cookie `core_console_session` is HttpOnly, `SameSite=Strict`, and `Secure` when `OAC_WEB_ORIGIN` is HTTPS. It lasts 12 hours. +- The session cookie `core_console_session` is HttpOnly, `SameSite=Strict`, and `Secure` when `OAC_PUBLIC_URL` is HTTPS. It lasts 12 hours. - Sessions live only in the console's memory, at most 64 at a time; the oldest is dropped first. A console restart or a Core key rotation signs everyone out. - At most two sign-in checks run at once; another attempt gets 429 with `Retry-After: 1`. - Failed attempts share a budget of 10 per minute; beyond it, a wrong key gets 429 with `Retry-After: 60`. The correct key always signs in, which is why the console refuses to start with a Core key shorter than 32 characters. @@ -98,7 +98,7 @@ With `OAC_WEB_NODE_PAYLOAD_DIR` set, the console serves the matched distribution ## Public address -The console does not configure a domain or obtain certificates. The operator's reverse proxy or hosting platform terminates HTTPS and routes to the console, and `OAC_PUBLIC_URL` records the origin that applications, nodes and executors use. The console accepts only the host of `OAC_WEB_ORIGIN`, so DNS rebinding cannot reach it. +The console does not configure a domain or obtain certificates. The operator's reverse proxy or hosting platform terminates HTTPS and routes to the console, and `OAC_PUBLIC_URL` records the origin that browsers, applications, nodes and executors use. The console accepts only its host, so DNS rebinding cannot reach it. ## Verification diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 79465aa1f..a8c1afe1c 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,19 +1,20 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 8eeef9a9742c5fd8a0bf27a5a31870a77dec59a436518dbe9d34770527c021a2 +source_hash: b0946948a110778123f52e065f24da9cfa7cfb8a1aeb6321526fb5064537a9a8 --- -Core 安装的每项设置都恰好只有一个归属位置。共有两类: +Core 安装的每项设置都恰好只有一个归属位置,分属以下三类: -| 类型 | 示例 | 归属位置 | 修改方式 | 生效方式 | +| 类别 | 示例 | 归属位置 | 修改方式 | 生效方式 | | --- | --- | --- | --- | --- | -| [进程设置](#process-settings-configjson) | 公共 URL、端口、日志、Harness、执行并发度、审计保留期、OAuth 来源、Runtime 历史记录导出 | 安装目录中的 `.env`(默认 `~/.oac/core`) | 编辑 `.env`,然后运行 `oac apply` | `oac apply` 会重新创建读取了这些已更改设置的服务 | +| [进程设置](#process-settings) | 公共 URL、端口、日志、Harness、执行并发度、审计保留期、OAuth 来源、Runtime 历史记录 | 安装目录中的 `.env`(默认 `~/.oac/core`) | 编辑 `.env`,然后运行 `oac apply` | `oac apply` 会重新创建读取了这些已更改设置的服务 | +| [机密信息](#compose-installations) | 数据库密码、凭据加密密钥、安装 ID、Core 密钥及由其派生的 Core 密钥摘要 | Compose 数据卷中的 `secrets/`,每项一份 | 初始化时一次性生成;`oac rotate-core-key` 替换 Core 密钥及其摘要 | `oac rotate-core-key` 会重启 Core 和 Web | | [运行时设置](#runtime-settings-web) | 沙箱后端和大小、节点、项目和密钥、默认模型、执行器凭据 | Core 的 PostgreSQL 数据库 | 在 Web 中修改,或使用 Core 密钥调用 Core API(`/core/v1`) | 保存时无需重启 Core;节点会异步准备 Runtime 变更 | -Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置,并在 **Startup settings** 下以只读方式显示 Core 加载的进程设置。机密信息存放在 [`secrets/`](#compose-installations) 中,每项仅保存一份。没有任何配置文件定义项目或 API 密钥。 +Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置,并在 **Startup settings** 下以只读方式显示 Core 加载的进程设置。没有任何配置文件定义项目或 API 密钥。 -## 进程设置 {#process-settings-configjson} +## 进程设置 {#process-settings} [安装选项](getting-started/install-options.md)中的安装标志只会一次性写入 `.env`。要更改设置,请编辑 `.env` 并应用: @@ -40,13 +41,13 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 ### 设置 {#settings} -`OAC_HISTORY_SETTINGS_FILE` 可以指向一个文件,其 headers 中含有导出凭据。该文件权限为 `0600`,这些 headers 绝不会出现在 `oac` 输出或安装报告中。模型提供商不属于进程设置;请参阅[默认模型](#default-models)。 +模型提供商不属于进程设置;请参阅[默认模型](#default-models)。 以下配置参考表保留英文原文。 | Variable | Default | Meaning | | --- | --- | --- | -| `OAC_PUBLIC_URL` | `http://localhost:8080` | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[修改公开 URL](#changing-the-public-url) | +| `OAC_PUBLIC_URL` | `http://localhost:8080`,由 `compose.yaml` 设置。未设置时启动的 Core 不运行 Runtime 网关,也不执行任何 Session | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[更改公共 URL](#changing-the-public-url) | | `OAC_HOST` | `127.0.0.1` | `compose.yaml` 发布的 Web 绑定地址。安装器设置为 `0.0.0.0` | | `OAC_WEB_PORT` | `8080` | Host port of Web | | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` | @@ -57,9 +58,23 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `OAC_HARNESSES` | Every registered Harness | Comma-separated Harnesses to enable besides the default one. Unknown names stop startup | | `OAC_WRITE_AUDIT_RETENTION` | `2160h` | Minimum `1h` | | `OAC_OAUTH_TRUSTED_ORIGINS` | unset | Comma-separated HTTPS origins | -| `OAC_HISTORY_SETTINGS_FILE` | unset | Optional Runtime history file. Sensitive; Core reports only whether it is configured | +| `OAC_HISTORY_SETTINGS_FILE` | unset | 可选的 [Runtime 历史文件](#runtime-history-file)。敏感;Core 只报告它是否已配置 | -未设置或为空的值使用默认值。编辑 `.env`,然后运行 `oac apply`。Core 会在 `GET /core/v1/installation` 报告它加载的进程设置。`oac-core check-config` 会在不启动 Core 的情况下校验同一组环境变量。敏感设置只报告是否已配置。有关 Core 如何收集和保留 Runtime 历史记录,请参阅[保留的历史记录](../../contracts/agents-api/zh/runtime-observability.md#retained-history-and-optional-export)。 +未设置或为空的值使用默认值。编辑 `.env`,然后运行 `oac apply`。Core 在启动时一次性读取所有进程设置及设置指向的文件,并在 `GET /core/v1/installation` 报告加载的结果。`oac-core check-config` 会在不启动 Core 的情况下加载并校验同样的设置和文件。`OAC_PROVIDER_ROOT` 下的原生安装程序目录清单不属于设置,Core 只在启动时检查它。错误信息只指明变量名,绝不包含其值。敏感设置只报告是否已配置。 + +### Runtime 历史文件 {#runtime-history-file} + +`OAC_HISTORY_SETTINGS_FILE` 指向一个 JSON 文件,用于调整[保留的历史记录](../../contracts/agents-api/zh/runtime-observability.md#retained-history-and-optional-export),并可添加 OTLP 导出。没有此文件时,Core 按下表默认值把历史记录保存在数据库中。在 Compose 安装中,把该文件放在数据卷的 `secrets/core/` 目录中,属主为 UID 65532,权限为 `0600`,并设置 `OAC_HISTORY_SETTINGS_FILE=/run/oac/`:Core 以只读方式把该目录挂载到 `/run/oac`。headers 中可以包含导出凭据,它们绝不会出现在 `oac` 输出或安装报告中。未知字段会被拒绝。 + +| 字段 | 默认值 | 含义 | +| --- | --- | --- | +| `sample_interval_seconds` | `30` | 定期采样间隔,范围为 5 到 300 | +| `queue_capacity` | `256` | 每个导出器排队的记录数,最大 4096 | +| `timeout_seconds` | `2` | 导出和历史查询超时,最大 30 | +| `endpoint` | 未设置 | 绝对 OTLP/HTTP 指标 URL,例如 `https://collector.example/v1/metrics`。未设置时 Core 不导出,其他导出字段也必须未设置 | +| `transport` | 未设置 | `otlp_http`;设置 `endpoint` 时必填 | +| `insecure` | `false` | `http` 端点必须设为 `true`,`https` 端点不允许设为 `true` | +| `headers` | 无 | 发往端点的请求标头。`Host`、`Content-Length`、`Content-Type` 和 `Content-Encoding` 为保留标头 | ## 运行时设置:Web {#runtime-settings-web} @@ -88,7 +103,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 ## Compose 安装 {#compose-installations} -发行版中的[独立 Compose 文件](getting-started/install-options.md#docker-compose-and-hosting-platforms)从平台环境读取进程设置。`OAC_PUBLIC_URL` 为空时选用 `http://localhost:8080`。把它设成准确的公共源地址,不要带尾部斜杠,并在添加节点或执行器之前重新创建 Core 和 Web。平台终止 TLS,并把流量转到 `web:8080`。 +发行版中的[独立 Compose 文件](getting-started/install-options.md#docker-compose-and-hosting-platforms)从平台环境读取进程设置。把 [`OAC_PUBLIC_URL`](#settings) 设成准确的公共源地址,不要带尾部斜杠,并在添加节点或执行器之前重新创建 Core 和 Web。平台终止 TLS,并把流量转到 `web:8080`。 初始化服务首次生成机密信息和安装 ID,随后在后续部署中验证它们。每项机密信息都只有一个持久来源;Core 的密钥摘要派生自 Web 的登录密钥。对于现有安装,初始化绝不会替换缺失或已更改的机密信息。Core 从 `.env` 读取进程环境。 @@ -98,7 +113,7 @@ Web 的 **System** 页面显示该安装的地址、默认模型和沙箱配置 | `secrets/database/` | 生成的数据库密码 | PostgreSQL 和 Core | | `secrets/core/` | 凭据加密密钥、安装 ID 和 Core 密钥摘要 | Core | | `secrets/web/` | 生成的 Core 登录密钥 | Web | -| `state/` | 私有 Provider 状态 | Core | +| `state/` | 私有 Provider 状态,在 Core 中挂载到 `/state`。每个适配器拥有一个子目录;E2B 使用 `e2b/`,不允许组或其他用户访问 | Core | | `node-payload/` | 已验证的节点安装元数据 | Web | 初始化会准备该目录;应用服务以只读方式接收各自的机密目录。`docker compose exec web oac-web core-key` 把 Core 密钥打印到运维人员终端,不写入容器日志。数据库密码和凭据加密密钥绝不打印。 @@ -139,21 +154,17 @@ Core 读取进程环境。Compose 将 `.env` 插值到环境中,并把机密 | 变量 | 设置来源 | | --- | --- | -| `OAC_PUBLIC_URL` | `public_url`,或 Core 的回环源地址。Core 从中派生守护进程 WebSocket URL、自托管 `remote_url`、托管沙箱地址和部署的只读 `core_url`,绝不从请求标头派生。未设置时,Core 不运行 Runtime 网关,也不执行任何 Session | +| `OAC_PUBLIC_URL` | [公共 URL](#settings)。Core 只校验一次,并从中派生 Agents API 基地址、守护进程 WebSocket URL、自托管 `remote_url`、安装程序下载地址、托管沙箱地址和部署的只读 `core_url`,绝不从请求标头派生 | | `OAC_ADDR` | 安装程序在容器中设置为 `:8091`。独立启动的 Core 在未设置或为空时,默认使用 `127.0.0.1:8091` | -| `OAC_DATABASE_URL` | 该安装不含密码的 PostgreSQL URL,并将 `core.database_pool` 作为 `pool_*` 查询参数附加到其中 | +| `OAC_DATABASE_URL` | 必填。不含密码的 PostgreSQL URL | | `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`。此时 URL 不得包含密码 | | `OAC_CREDENTIAL_KEY_FILE` | `/run/oac/credential.key` | -| `OAC_CORE_KEY_DIGESTS_FILE` | `/run/oac/core-key-digests.json`:一个包含 Core 密钥 SHA-256 的 JSON 数组 | -| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`:安装 ID,采用规范 UUID 格式。它会启用沙箱部署和节点路由,并要求设置 `OAC_PUBLIC_URL` 和 `OAC_CORE_KEY_DIGESTS_FILE`。如果 ID 与数据库记录的 ID 不一致,Core 会拒绝它,因此必须将两者一同保留 | -| `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS` | 对应的[进程设置](#settings)。`oac-core check-config` 会在不启动 Core 的情况下校验它们 | -| `OAC_HISTORY_SETTINGS_FILE` | 可选的 Runtime 历史文件。敏感;安装报告只说明它是否已设置 | -| `OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | `log.*`;Web 也读取这三个设置 | -| `OAC_PROVIDER_ROOT` | 适配器构件的绝对根目录。Core 镜像设置为 `/opt/oac`。每个适配器都拥有此根目录下的辅助路径 | -| `OAC_PROVIDER_STATE_ROOT` | 绝对私有状态根目录:Core 镜像中为 `/state`。每个适配器都拥有自己的子目录;E2B 使用 `e2b/`,该目录归 Core 的用户所有,不允许组或其他用户访问。将其与数据库和 `credential.key` 一起备份;不要将其挂载到 Web 或 Runtime 中 | -| `OAC_NATIVE_INSTALLER_DIR` | 自托管守护进程安装程序:Compose 文件中为 `/opt/oac/native-installers`。未设置时 Core 不提供安装程序。提供目录清单前,Core 会将其与自身发行版进行核对 | +| `OAC_CORE_KEY_DIGESTS_FILE` | 必填。`/run/oac/core-key-digests.json`:一个包含 Core 密钥 SHA-256 的 JSON 数组 | +| `OAC_INSTALLATION_ID_FILE` | `/run/oac/installation.id`:安装 ID,采用规范 UUID 格式。它会启用沙箱部署和节点路由,并要求设置 `OAC_PUBLIC_URL`。如果 ID 与数据库记录的 ID 不一致,Core 会拒绝它 | +| `OAC_EXECUTION_CONCURRENCY`、`OAC_DEFAULT_HARNESS`、`OAC_HARNESSES`、`OAC_WRITE_AUDIT_RETENTION`、`OAC_OAUTH_TRUSTED_ORIGINS`、`OAC_HISTORY_SETTINGS_FILE`、`OAC_LOG_LEVEL`、`OAC_LOG_FORMAT`、`OAC_LOG_ADD_SOURCE` | 对应的[进程设置](#settings)。Web 也读取三个日志设置 | +| `OAC_PROVIDER_ROOT` | 适配器构件的绝对根目录。Core 镜像设置为 `/opt/oac`。每个适配器都拥有此根目录下的辅助路径。当其中的 `native-installers/` 目录包含 `catalog.json` 时,Core 在核对该目录清单与自身发行版后提供自托管守护进程安装程序。适配器状态位于 `/state`,即数据卷的 [`state/`](#compose-installations) | -Core 会记录所加载文件的路径,但绝不记录环境变量的值或文件内容。 +Core 会记录所加载的历史文件路径,但绝不记录环境变量的值或文件内容。 显式 OAuth 受信任源无效时,Core 会停止启动。条目必须是不含凭据、查询参数和非根路径的 HTTPS 源地址。[Vaults](../../contracts/agents-api/zh/vaults.md) 负责刷新和网络策略。私有颁发者还需要受信任的 CA:独立管理的 Unix Core 可以使用 Go 的 `SSL_CERT_FILE` PEM CA-bundle 覆盖机制,从而保留证书验证。托管安装没有自定义 CA 设置。 @@ -163,11 +174,11 @@ Compose 为 Web 设置这些变量。仅在不使用 Compose 运行控制台时 | 变量 | 默认值 | 含义 | | --- | --- | --- | -| `OAC_WEB_ADDR` | `:8080` | 监听地址 | -| `OAC_WEB_ORIGIN` | `http://127.0.0.1:8080` | 面向浏览器的准确源地址,可以使用 HTTP 或 HTTPS,且不得包含路径。Host 和源地址检查使用此值;HTTPS 会使 Session Cookie 具备 `Secure` 属性 | -| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core 的源地址,可以使用 HTTP 或 HTTPS,且不得包含凭据、查询参数或路径 | -| `OAC_WEB_CORE_KEY_FILE` | `/admin/core.key` | 常规文件的绝对路径,该文件没有组或其他用户权限,并保存 Core 密钥:至少 32 个字符、不含空白字符、最大 4 KiB | +| `OAC_WEB_ADDR` | `:8080` | 监听地址。主机部分为空或未指定时,健康检查在 `127.0.0.1` 上探测它 | +| `OAC_PUBLIC_URL` | 必填 | [公共 URL](#settings):面向浏览器的准确源地址,可以使用 HTTP 或 HTTPS,且不得包含路径。Host 和源地址检查使用此值;HTTPS 会使 Session Cookie 具备 `Secure` 属性 | +| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core 的源地址,可以使用 HTTP 或 HTTPS,且不得包含凭据、查询参数或路径。健康检查探测其 `/healthz` | +| `OAC_WEB_CORE_KEY_FILE` | 必填 | 常规文件的绝对路径,该文件没有组或其他用户权限,并保存 Core 密钥:至少 32 个字符、不含空白字符、最大 4 KiB | | `OAC_WEB_DIST` | `/www` | 已构建控制台的绝对目录;必须包含 `index.html` | | `OAC_WEB_NODE_PAYLOAD_DIR` | 未设置 | 所匹配发行版的节点载荷(即安装程序的 `node-payload/`)的绝对路径。未设置时,不提供 `/node-install/*`,且 Add node 不可用 | -变量不存在时会应用默认值;显式空值会按已提供的值进行验证。无效的 `OAC_WEB_*` 值会阻止控制台启动,并显示一条指明变量名的消息。控制台还会读取 `OAC_LOG_LEVEL`、`OAC_LOG_FORMAT` 和 `OAC_LOG_ADD_SOURCE`([Core 环境](#appendix-core-environment-without-the-installer));未知值会回退到其默认值。对于不在同一台计算机上的任何浏览器,请使用 HTTPS。 +未设置或为空的变量使用其默认值。无效值会阻止控制台启动,并显示一条指明变量名的消息。控制台还会读取三个日志[进程设置](#settings),并拒绝 Core 拒绝的值。对于不在同一台计算机上的任何浏览器,请使用 HTTPS。 diff --git a/docs/zh/web/console-server.md b/docs/zh/web/console-server.md index e7b67fd2a..a152253ce 100644 --- a/docs/zh/web/console-server.md +++ b/docs/zh/web/console-server.md @@ -1,7 +1,7 @@ --- title: "控制台服务器" source: docs/web/console-server.md -source_hash: b0302f0cf27ccd116c4bbb9477d5853f4ae1bf6cea34c9a4e21af72d25656557 +source_hash: 2cc4b562301d95640d2b653ec522a5407a70a98e1f4e3c1fe89bd246ffd1199e --- 控制台服务器(`services/web`、`oac-web` 进程)提供构建后的控制台,使用 Core 密钥认证管理员,并将已登录浏览器的 `/core/v1` 请求携带该密钥转发到 Core。浏览器不持有 Core 密钥或任何 API 密钥。应用、节点和自托管执行器经控制台到达 Core,控制台原样转发 `/v1`、`/api/v1` 和 `/docs`。 @@ -44,7 +44,7 @@ flowchart LR 除 `/healthz`、`/v1`、`/api/v1` 和 `/docs` 外,每个请求首先必须通过这些检查: -1. **Host 与来源。** `Host` 请求头必须等于 `OAC_WEB_ORIGIN` 的主机。存在 `Origin` 时必须等于该来源,`Sec-Fetch-Site` 必须为 `same-origin` 或 `none`。写请求既无 `Origin` 又无 `Sec-Fetch-Site: same-origin` 时,需要同源 `Referer`。否则控制台返回 403。`/node-install/*` 仅检查主机和路径。 +1. **Host 与来源。** `Host` 请求头必须等于 `OAC_PUBLIC_URL` 的主机。存在 `Origin` 时必须等于该来源,`Sec-Fetch-Site` 必须为 `same-origin` 或 `none`。写请求既无 `Origin` 又无 `Sec-Fetch-Site: same-origin` 时,需要同源 `Referer`。否则控制台返回 403。`/node-install/*` 仅检查主机和路径。 2. **安全请求。** 路径必须以 `/` 开头,不含 `%`、反斜杠、NUL、点路径段或空路径段。绝对形式请求目标、`CONNECT` 和 `TRACE` 返回 400。`Upgrade` 头返回 400,但 `/v1`、`/api/v1` 和 `/docs` 在这些检查之前就被转发。因此 `/core/v1` 请求无法离开该前缀。 3. **登录。** 需要登录的路径在无有效会话 cookie 时返回 401。 @@ -77,7 +77,7 @@ flowchart LR 管理员使用部署的 [Core 密钥](../getting-started/operations.md#core-key)登录。没有控制台账号、用户名或设置步骤,登录授予整个控制台访问权限。 - 控制台以恒定时间比较提交密钥与配置密钥的 SHA-256 摘要,不记录或返回密钥。 -- 会话 cookie `core_console_session` 为 HttpOnly、`SameSite=Strict`,`OAC_WEB_ORIGIN` 为 HTTPS 时还设置 `Secure`。有效期 12 小时。 +- 会话 cookie `core_console_session` 为 HttpOnly、`SameSite=Strict`,`OAC_PUBLIC_URL` 为 HTTPS 时还设置 `Secure`。有效期 12 小时。 - 会话仅存在控制台内存中,最多同时 64 个,先移除最旧的。重启控制台或轮换 Core 密钥会让所有用户退出登录。 - 同时最多执行两次登录检查;额外尝试返回 429 和 `Retry-After: 1`。 - 失败尝试共享每分钟 10 次预算;超出后,错误密钥返回 429 和 `Retry-After: 60`。正确密钥始终可以登录,因此控制台拒绝使用少于 32 字符的 Core 密钥启动。 @@ -100,7 +100,7 @@ flowchart LR ## 公开地址 {#public-address} -控制台不配置域名,也不申请证书。运维人员的反向代理或托管平台终止 HTTPS 并把流量转到控制台,`OAC_PUBLIC_URL` 记录应用、节点和执行器使用的源地址。控制台只接受 `OAC_WEB_ORIGIN` 的主机,因此 DNS 重绑定不能访问它。 +控制台不配置域名,也不申请证书。运维人员的反向代理或托管平台终止 HTTPS 并把流量转到控制台,`OAC_PUBLIC_URL` 记录浏览器、应用、节点和执行器使用的源地址。控制台只接受该地址的主机,因此 DNS 重绑定不能访问它。 ## 验证 {#verification} diff --git a/internal/obs/log/init.go b/internal/obs/log/init.go index d570e56be..10c5194dd 100644 --- a/internal/obs/log/init.go +++ b/internal/obs/log/init.go @@ -1,10 +1,10 @@ package log import ( + "errors" "io" "log/slog" "os" - "strings" "sync" ) @@ -22,22 +22,42 @@ type Config struct { Out io.Writer } -// ConfigFromEnv reads: +// LoadConfig reads the logging settings Core and Web share: // -// OAC_LOG_FORMAT = json | text (default: auto) // OAC_LOG_LEVEL = debug | info | warn | error (default: info) +// OAC_LOG_FORMAT = auto | json | text (default: auto) // OAC_LOG_ADD_SOURCE = 0 | 1 (default: 0) // -// Unknown values fall back to defaults — Init runs before most -// error-handling exists, so "boot anyway" beats "panic on typo". -func ConfigFromEnv() Config { - cfg := Config{ - Format: strings.ToLower(strings.TrimSpace(os.Getenv("OAC_LOG_FORMAT"))), - Level: parseLevel(os.Getenv("OAC_LOG_LEVEL")), - AddSource: os.Getenv("OAC_LOG_ADD_SOURCE") == "1", - Out: os.Stderr, +// Unset or empty selects the default. Any other value is an error that names +// the variable and never echoes the value. +func LoadConfig() (Config, error) { + var cfg Config + switch os.Getenv("OAC_LOG_LEVEL") { + case "", "info": + case "debug": + cfg.Level = slog.LevelDebug + case "warn": + cfg.Level = slog.LevelWarn + case "error": + cfg.Level = slog.LevelError + default: + return Config{}, errors.New("OAC_LOG_LEVEL must be debug, info, warn or error") + } + switch format := os.Getenv("OAC_LOG_FORMAT"); format { + case "", "auto": + case "json", "text": + cfg.Format = format + default: + return Config{}, errors.New("OAC_LOG_FORMAT must be auto, json or text") + } + switch os.Getenv("OAC_LOG_ADD_SOURCE") { + case "", "0": + case "1": + cfg.AddSource = true + default: + return Config{}, errors.New("OAC_LOG_ADD_SOURCE must be 0 or 1") } - return cfg + return cfg, nil } // isTerminal reports whether f is a character device (TTY) so the JSON @@ -53,19 +73,6 @@ func isTerminal(f *os.File) bool { return info.Mode()&os.ModeCharDevice != 0 } -func parseLevel(s string) slog.Level { - switch strings.ToLower(strings.TrimSpace(s)) { - case "debug": - return slog.LevelDebug - case "warn", "warning": - return slog.LevelWarn - case "error", "err": - return slog.LevelError - default: - return slog.LevelInfo - } -} - // initOnce guarantees Init's slog.SetDefault side-effect runs at most // once per process so tests don't fight over the global handler. var initOnce sync.Once diff --git a/internal/obs/log/init_test.go b/internal/obs/log/init_test.go new file mode 100644 index 000000000..1d7fd7c4c --- /dev/null +++ b/internal/obs/log/init_test.go @@ -0,0 +1,30 @@ +package log + +import ( + "log/slog" + "strings" + "testing" +) + +func TestLoadConfigIsStrictAndEmptyMeansDefault(t *testing.T) { + t.Setenv("OAC_LOG_LEVEL", "") + t.Setenv("OAC_LOG_FORMAT", "auto") + t.Setenv("OAC_LOG_ADD_SOURCE", "") + if cfg, err := LoadConfig(); err != nil || cfg.Level != slog.LevelInfo || cfg.Format != "" || cfg.AddSource { + t.Fatal(cfg, err) + } + t.Setenv("OAC_LOG_LEVEL", "warn") + t.Setenv("OAC_LOG_FORMAT", "text") + t.Setenv("OAC_LOG_ADD_SOURCE", "1") + if cfg, err := LoadConfig(); err != nil || cfg.Level != slog.LevelWarn || cfg.Format != "text" || !cfg.AddSource { + t.Fatal(cfg, err) + } + for name, value := range map[string]string{"OAC_LOG_LEVEL": "warning", "OAC_LOG_FORMAT": "JSON", "OAC_LOG_ADD_SOURCE": "true"} { + t.Run(name, func(t *testing.T) { + t.Setenv(name, value) + if _, err := LoadConfig(); err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), value) { + t.Fatal(err) + } + }) + } +} diff --git a/services/core/IMPLEMENTATION.md b/services/core/IMPLEMENTATION.md index 47031e10a..e609040ff 100644 --- a/services/core/IMPLEMENTATION.md +++ b/services/core/IMPLEMENTATION.md @@ -138,7 +138,7 @@ Provider input validation uses the adapter rules in `internal/harnessconfig`: on - At dispatch, Core rechecks the tenant, attached Vault, selected ID, frozen auth type and exact URL before scoped decryption, and the token enters only the transient daemon request. A missing key or binding failure never falls back to anonymous execution. - `credentialcrypto` ciphertext is a format version byte followed by the standard AEAD nonce, ciphertext and tag. The authenticated data holds a fixed domain and version plus the binding (tenant, Vault, Credential, auth type, exact destination). Keep the domain string unchanged: existing rows must still decrypt. -- Random-nonce GCM allows at most 2^32 encryptions per key. `secrets/credential.key` also seals model providers, the E2B key, Skills, initial files and environment setup, so every sealed write counts toward that bound; there is no rotation or re-encryption path. +- Random-nonce GCM allows at most 2^32 encryptions per key. `secrets/core/credential.key` also seals model providers, the E2B key, Skills, initial files and environment setup, so every sealed write counts toward that bound; there is no rotation or re-encryption path. - OAuth dispatch refresh holds the Credential row lock and the external exchange under one 20-second context (`vaults.oauthRefreshTimeout`). The refresh HTTP client has a 10-second overall timeout and 5-second TLS handshake and response-header timeouts, uses no proxy and treats any redirect as failure. ## MCP diff --git a/services/core/cmd/oac/main.go b/services/core/cmd/oac/main.go index f2b3393d3..69e15bad7 100644 --- a/services/core/cmd/oac/main.go +++ b/services/core/cmd/oac/main.go @@ -25,7 +25,7 @@ func main() { os.Exit(2) } if os.Args[1] == "init" { - log.Init(log.ConfigFromEnv()) + log.Init(log.Config{}) } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -80,9 +80,6 @@ func run(ctx context.Context, command string, args []string) error { } func installDir() (string, error) { - if dir := os.Getenv("OAC_INSTALL_DIR"); dir != "" { - return dir, nil - } exe, err := os.Executable() if err != nil { return "", err diff --git a/services/core/cmd/server/core_metrics.go b/services/core/cmd/server/core_metrics.go index 84d4c59a4..52f81462a 100644 --- a/services/core/cmd/server/core_metrics.go +++ b/services/core/cmd/server/core_metrics.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/coremetricspg" @@ -74,14 +75,19 @@ func (s *coreMetricsSource) History(ctx context.Context, start, end time.Time, s return s.store.ReadExecutionHistory(ctx, start, end, step) } -func reportCleanupResult(metrics *coremetrics.Service, job string, count int64, err error) { - if metrics == nil { - return - } - processed, failed := &count, int64(0) - if err != nil { - processed = nil - failed = 1 - } - metrics.ReportJob(job, time.Now(), processed, &failed, err) +// prune makes a retention pass, bounded by timeout, a job that runs every +// minute. A failed pass counts no rows and one failure. +func prune(id string, timeout time.Duration, run func(context.Context) (int64, error)) coremetrics.Periodic { + return coremetrics.Periodic{ID: id, Every: time.Minute, Run: func(ctx context.Context) (*int64, int64, error) { + pass, cancel := context.WithTimeout(ctx, timeout) + count, err := run(pass) + cancel() + if err != nil { + if ctx.Err() == nil { + log.Ctx(ctx).Warn("Retention cleanup failed", "job", id) + } + return nil, 1, err + } + return &count, 0, nil + }} } diff --git a/services/core/cmd/server/credential_cipher.go b/services/core/cmd/server/credential_cipher.go deleted file mode 100644 index 60a6754c7..000000000 --- a/services/core/cmd/server/credential_cipher.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "encoding/base64" - "errors" - "os" - "strings" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" -) - -func credentialCipher() (*credentialcrypto.Cipher, error) { - path := os.Getenv("OAC_CREDENTIAL_KEY_FILE") - if path == "" { - return nil, nil - } - content, err := os.ReadFile(path) - if err != nil { - return nil, errors.New("cannot read OAC_CREDENTIAL_KEY_FILE") - } - key, err := base64.StdEncoding.Strict().DecodeString(strings.TrimSpace(string(content))) - if err != nil || len(key) != 32 { - return nil, errors.New("OAC_CREDENTIAL_KEY_FILE must contain a base64-encoded random 32-byte key") - } - return credentialcrypto.New(key) -} diff --git a/services/core/cmd/server/credential_cipher_test.go b/services/core/cmd/server/credential_cipher_test.go deleted file mode 100644 index 801d04c91..000000000 --- a/services/core/cmd/server/credential_cipher_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "bytes" - "encoding/base64" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" -) - -func TestCredentialCipherConfiguration(t *testing.T) { - t.Setenv("OAC_CREDENTIAL_KEY_FILE", "") - if c, err := credentialCipher(); c != nil || err != nil { - t.Fatal("absent dedicated key must remain disabled", err) - } - path := filepath.Join(t.TempDir(), "credential.key") - t.Setenv("OAC_CREDENTIAL_KEY_FILE", path) - if _, err := credentialCipher(); err == nil { - t.Fatal("missing configured file accepted") - } - for _, content := range []string{"", "private-invalid-key", base64.StdEncoding.EncodeToString(make([]byte, 31))} { - if err := os.WriteFile(path, []byte(content), 0600); err != nil { - t.Fatal(err) - } - if _, err := credentialCipher(); err == nil || strings.Contains(err.Error(), "private-invalid-key") { - t.Fatal("invalid configuration accepted or leaked") - } - } - key := bytes.Repeat([]byte{0x91}, 32) - if err := os.WriteFile(path, []byte(base64.StdEncoding.EncodeToString(key)+"\n"), 0600); err != nil { - t.Fatal(err) - } - first, err := credentialCipher() - if err != nil { - t.Fatal(err) - } - binding := credentialcrypto.Binding{TenantID: "tenant", VaultID: "vault", CredentialID: "credential", AuthType: "static_bearer", Destination: "https://example.invalid/mcp"} - sealed, err := first.Seal([]byte("opaque storage test"), binding) - if err != nil { - t.Fatal(err) - } - reopened, err := credentialCipher() - if err != nil { - t.Fatal(err) - } - got, err := reopened.Open(sealed, binding) - if err != nil || string(got) != "opaque storage test" { - t.Fatal("persisted key did not recover ciphertext", err) - } -} diff --git a/services/core/cmd/server/daemon_bootstrap.go b/services/core/cmd/server/daemon_bootstrap.go deleted file mode 100644 index d4c0644ea..000000000 --- a/services/core/cmd/server/daemon_bootstrap.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "errors" - "net/url" -) - -// runtimeWebSocketURL derives the daemon WebSocket URL from a Core origin or -// its /api/v1 base. -func runtimeWebSocketURL(coreURL string) (string, error) { - u, err := url.Parse(coreURL) - if err != nil || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { - return "", errors.New("managed Runtime Core address is unavailable") - } - switch u.Scheme { - case "https": - u.Scheme = "wss" - case "http": - u.Scheme = "ws" - default: - return "", errors.New("managed Runtime Core address is unavailable") - } - u.Path = "/api/v1/agent-daemon/ws" - return u.String(), nil -} diff --git a/services/core/cmd/server/installation.go b/services/core/cmd/server/installation.go index a927a4e3c..11097db7f 100644 --- a/services/core/cmd/server/installation.go +++ b/services/core/cmd/server/installation.go @@ -12,27 +12,18 @@ var sourceCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) // installationFacts reports what GET /core/v1/installation serves: Core's own // environment and build, plus the process settings it loaded. -func installationFacts(publicURL string) (api.Installation, error) { - var facts api.Installation - id, err := processconfig.InstallationID() - if err != nil { - return facts, err +func installationFacts(config processconfig.Config) api.Installation { + facts := api.Installation{Configuration: api.InstallationConfiguration{Settings: config.Settings()}} + if config.InstallationID != "" { + facts.InstallationID = &config.InstallationID } - if id != "" { - facts.InstallationID = &id - } - if publicURL != "" { - base := publicURL + "/v1" - facts.PublicURL, facts.APIBaseURL, facts.LocalOnly = &publicURL, &base, placement.LoopbackOrigin(publicURL) + if origin := config.PublicOrigin; origin != nil { + public, base := origin.String(), origin.API() + facts.PublicURL, facts.APIBaseURL, facts.LocalOnly = &public, &base, placement.LoopbackOrigin(public) } if sourceCommit.MatchString(buildRevision) { revision := buildRevision facts.SourceCommit = &revision } - settings, err := processconfig.Settings() - if err != nil { - return facts, err - } - facts.Configuration = api.InstallationConfiguration{Settings: settings} - return facts, nil + return facts } diff --git a/services/core/cmd/server/main.go b/services/core/cmd/server/main.go index f734986f2..a297e4e58 100644 --- a/services/core/cmd/server/main.go +++ b/services/core/cmd/server/main.go @@ -35,7 +35,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/agents" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/databaseurl" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/environmenttemplates" @@ -43,6 +42,7 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/files" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/modelconfiguration" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/nativeinstaller" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/oauthrefresh" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/agentpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/auditpg" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/coremetricspg" @@ -71,53 +71,39 @@ import ( ) func main() { + config, err := processconfig.Load() if len(os.Args) > 1 && os.Args[1] == "check-config" { - if err := processconfig.Check(); err != nil { + if err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } return } - if err := run(); err != nil { + if err == nil { + err = run(config) + } + if err != nil { log.Bg().Error("oac-core startup failed", "error", err) os.Exit(1) } } -func run() error { - if err := processconfig.Check(); err != nil { - return err - } - log.Init(log.ConfigFromEnv()) - public, err := processconfig.PublicURL() - if err != nil { - return err - } - concurrency, err := processconfig.ExecutionConcurrency() - if err != nil { - return err - } - logConfigurationSources() - databaseURL, err := databaseurl.FromEnvironment() - if err != nil { - return err - } - if databaseURL == "" { - return errors.New("OAC_DATABASE_URL is required") - } - credentialKey, err := credentialCipher() - if err != nil { - return err +func run(config processconfig.Config) error { + log.Init(config.Log) + log.Bg().Info("Core process configuration loaded from the process environment") + if file := config.RuntimeHistory.File; file != "" { + log.Bg().Info("Core auxiliary configuration", "setting", "OAC_HISTORY_SETTINGS_FILE", "path", file) } + credentialKey := config.CredentialKey ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() migrating, cancelMigration := context.WithTimeout(ctx, 2*time.Minute) - err = migrations.Apply(migrating, databaseURL) + err := migrations.Apply(migrating, config.DatabaseURL) cancelMigration() if err != nil { return fmt.Errorf("Agents API database migration failed: %w", err) } - pool, err := pgxpool.New(ctx, databaseURL) + pool, err := pgxpool.New(ctx, config.DatabaseURL) if err != nil { return errors.New("invalid Agents API database configuration") } @@ -127,15 +113,7 @@ func run() error { if err := pool.Ping(ready); err != nil { return errors.New("Agents API database connection failed") } - engine, err := processconfig.DefaultHarness() - if err != nil { - return err - } - kinds, err := processconfig.Harnesses(engine) - if err != nil { - return err - } - oauthClient, err := oauthRefreshClient() + oauthClient, err := oauthrefresh.NewClient(config.OAuthTrustedOrigins) if err != nil { return err } @@ -172,6 +150,10 @@ func run() error { return err } sandboxProviders := providers.Builtin() + var public string + if config.PublicOrigin != nil { + public = config.PublicOrigin.String() + } // The placement rules are built once: the provider declarations and the // public URL never change while Core runs. placementRules, err := placement.NewRules(sandboxProviders, public) @@ -188,34 +170,14 @@ func run() error { if err != nil { return err } - installation, err := installationFacts(public) - if err != nil { - return err - } - metricsSource := &coreMetricsSource{store: coremetricspg.New(units), pool: pool} - metrics := coremetrics.New(processStartedAt, buildRevision, metricsSource) - auditRetention, err := writeAuditRetention() - if err != nil { - return err - } - auditCleanupCtx, cancelAuditCleanup := context.WithCancel(ctx) - auditCleanupDone := make(chan struct{}) - go func() { - defer close(auditCleanupDone) - runWriteAuditCleanup(auditCleanupCtx, auditStore, auditRetention, metrics) - }() - defer func() { cancelAuditCleanup(); <-auditCleanupDone }() var workerDone chan error var worker *execution.Worker - managedNodes, err := configureManagedNodes(deploymentService, deploymentStore, sandboxProviders, public, func(ctx context.Context) error { + managedNodes := configureManagedNodes(deploymentService, deploymentStore, sandboxProviders, config, func(ctx context.Context) error { if worker == nil { return errors.New("sandbox execution owner is unavailable") } return worker.CheckOwnership(ctx) }) - if err != nil { - return err - } defer managedNodes.close() var managed *execution.RuntimeProvider observationSources := map[string]runtimeobs.SourceResolver{} @@ -227,7 +189,7 @@ func run() error { if err != nil { return err } - history, err := runtimeHistory(ctx, units, public != "") + history, err := runtimeHistory(ctx, units, config.RuntimeHistory, config.PublicOrigin != nil) if err != nil { return err } @@ -248,23 +210,7 @@ func run() error { closeRuntimeHistory(closeCtx, history.Exporter) } }() - cleanupCtx, cancelCleanup := context.WithCancel(ctx) - cleanupDone := make(chan struct{}) - go func() { - defer close(cleanupDone) - runHistoryCleanup(cleanupCtx, history.Prune, metrics) - }() - defer func() { cancelCleanup(); <-cleanupDone }() - var keyAdmin *api.DeploymentAuthenticator - if managedNodes != nil { - keyAdmin = managedNodes.admin - } else { - keyAdmin, err = deploymentAdminAuthenticator() - if err != nil { - return err - } - } - if err := api.ValidateCredentialSeparation(ctx, keyAdmin, projectStore); err != nil { + if err := api.ValidateCredentialSeparation(ctx, config.CoreKeys, projectStore); err != nil { return err } historyService, err := runtimehistory.NewService(sessionStore, history.Reader) @@ -275,25 +221,22 @@ func run() error { var registry *runtimegateway.Registry var executorURL string var nativeInstaller *api.NativeInstaller - if public != "" { - executorURL, err = runtimeWebSocketURL(public) - if err != nil { - return err - } + if origin := config.PublicOrigin; origin != nil { + executorURL = origin.DaemonWebSocket() daemonHandler, registry, err = runtime.NewGateway(sessionStore, sessionService, sessionStore, executorURL) if err != nil { return err } defer runtime.CloseConnections(registry) var catalog *nativeinstaller.Catalog - if directory := os.Getenv("OAC_NATIVE_INSTALLER_DIR"); directory != "" { - catalog, err = nativeinstaller.Load(directory, buildRevision) + if config.NativeInstallers != "" { + catalog, err = nativeinstaller.Load(config.NativeInstallers, buildRevision) if err != nil { return err } } if buildRevision != "" { - nativeInstaller = &api.NativeInstaller{Version: buildRevision, Catalog: catalog} + nativeInstaller = &api.NativeInstaller{Version: buildRevision, Base: origin.InstallerBase(), Catalog: catalog} } } var deploymentExecution *deployment.ExecutionOperations @@ -302,7 +245,7 @@ func run() error { Credentials: vaultService, Observer: modelConfigurationStore, Deployment: deploymentService, DeploymentReader: deploymentStore, Sessions: sessionService, SessionsReader: sessionStore, - ManagedRuntimes: managed, MaxConcurrentExecutions: concurrency} + ManagedRuntimes: managed, MaxConcurrentExecutions: config.ExecutionConcurrency} lease, err := pgunit.AcquireLease(ctx, pool) if err != nil { return err @@ -333,46 +276,40 @@ func run() error { } }() } - if history.SampleInterval == 0 { - metrics.StopJob("runtime_sampler") - } - if history.SampleInterval > 0 { - if worker == nil { - return errors.New("Runtime history periodic sampling requires the execution worker") - } - sampler, err := runtimeobs.NewSampler(observationResolver, observationService, worker, runtimeobs.SamplerOptions{ - Interval: history.SampleInterval, - Report: func(result runtimeobs.SweepResult) { - sampleCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - sampleErr := worker.CheckOwnership(sampleCtx) - if sampleErr == nil { - _, sampleErr = deploymentStore.SampleHostHistory(sampleCtx) - } - cancel() - if !result.Complete { - sampleErr = errors.New("incomplete Runtime sampling sweep") - } - metrics.ReportJob("runtime_sampler", result.CompletedAt, metricPtr(int64(result.Observed)), metricPtr(int64(result.Failed)), sampleErr) - fields := []any{"listed", result.Listed, "observed", result.Observed, "failed", result.Failed, "complete", result.Complete} - if result.Complete { - log.Bg().Debug("Runtime history sampling sweep complete", fields...) - } else { - log.Bg().Warn("Runtime history sampling sweep incomplete", fields...) - } - }, - }) + // Sampling runs only with the Worker, which owns every sweep. + sampling := coremetrics.Periodic{ID: "runtime_sampler", Every: history.SampleInterval} + if worker != nil { + sampler, err := runtimeobs.NewSampler(observationResolver, observationService, worker, runtimeobs.SamplerOptions{}) if err != nil { return err } - samplerCtx, cancelSampler := context.WithCancel(ctx) - samplerDone := make(chan error, 1) - go func() { defer metrics.StopJob("runtime_sampler"); samplerDone <- sampler.Run(samplerCtx) }() - defer func() { - cancelSampler() - <-samplerDone - }() + sampling.Run = func(ctx context.Context) (*int64, int64, error) { + result := sampler.Sweep(ctx) + sampleCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + err := worker.CheckOwnership(sampleCtx) + if err == nil { + _, err = deploymentStore.SampleHostHistory(sampleCtx) + } + cancel() + fields := []any{"listed", result.Listed, "observed", result.Observed, "failed", result.Failed, "complete", result.Complete} + if !result.Complete { + log.Bg().Warn("Runtime history sampling sweep incomplete", fields...) + err = errors.New("incomplete Runtime sampling sweep") + } else { + log.Bg().Debug("Runtime history sampling sweep complete", fields...) + } + return metricPtr(int64(result.Observed)), int64(result.Failed), err + } + } + metricsSource := &coreMetricsSource{store: coremetricspg.New(units), pool: pool, worker: worker, registry: registry} + metrics, err := coremetrics.New(processStartedAt, buildRevision, metricsSource, sampling, + prune("history_cleanup", 2*time.Second, history.Prune), + prune("audit_cleanup", 5*time.Second, func(ctx context.Context) (int64, error) { + return auditStore.DeleteExpiredWriteOperations(ctx, time.Now().Add(-config.WriteAuditRetention), 1000) + })) + if err != nil { + return err } - metricsSource.worker, metricsSource.registry = worker, registry metricsCtx, cancelMetrics := context.WithCancel(ctx) metricsDone := make(chan struct{}) go func() { defer close(metricsDone); metrics.Run(metricsCtx) }() @@ -383,8 +320,8 @@ func run() error { return err } deps := api.Dependencies{ - Engine: engine, Harnesses: kinds, CoreKeys: keyAdmin, - Installation: installation, InstallationBindings: deploymentService, + Engine: config.DefaultHarness, Harnesses: config.Harnesses, CoreKeys: config.CoreKeys, + Installation: installationFacts(config), InstallationBindings: deploymentService, Projects: projectService, ProjectsReader: projectStore, ModelProviders: modelConfigurationService, ModelProvidersReader: modelConfigurationStore, Vaults: vaultService, VaultsReader: vaultStore, @@ -438,8 +375,7 @@ func run() error { } handler = serverHandler(handler, &routes) } - addr := serverAddress() - server := &http.Server{Addr: addr, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second} + server := &http.Server{Addr: config.Addr, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second} done := make(chan error, 1) go func() { done <- server.ListenAndServe() }() select { diff --git a/services/core/cmd/server/managed_nodes.go b/services/core/cmd/server/managed_nodes.go index 8e9add39e..a6d8f434b 100644 --- a/services/core/cmd/server/managed_nodes.go +++ b/services/core/cmd/server/managed_nodes.go @@ -2,11 +2,8 @@ package main import ( "context" - "encoding/json" "errors" - "os" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" @@ -17,40 +14,21 @@ import ( ) type managedNodes struct { - setup *managedSetup - runtime *execution.RuntimeProvider - hub *node.Hub - admin *api.DeploymentAuthenticator - closeProvider func() + setup *managedSetup + runtime *execution.RuntimeProvider + hub *node.Hub } // configureManagedNodes serves the nodes of the Web-managed deployment. Node // presence and health and the generation of each allocation go through the // deployment service; the owner epoch that fences connections and the -// allocations each generation retains are read from the deployment reader. -func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, registry *providers.Registry, publicURL string, owner func(context.Context) error) (*managedNodes, error) { - setupID, err := processconfig.InstallationID() - if err != nil || setupID == "" { - return nil, err - } - if publicURL == "" { - return nil, errors.New("OAC_INSTALLATION_ID_FILE requires OAC_PUBLIC_URL, the origin nodes and sandboxes use to reach Core") - } - closeProvider := func() {} - result := &managedNodes{closeProvider: closeProvider} - success := false - defer func() { - if !success { - closeProvider() - } - }() - result.admin, err = deploymentAdminAuthenticator() - if err != nil { - return nil, err - } - if result.admin == nil { - return nil, errors.New("Web sandbox setup requires OAC_CORE_KEY_DIGESTS_FILE with the Core key digest") +// allocations each generation retains are read from the deployment reader. It +// returns nil without an installation ID. +func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, registry *providers.Registry, config processconfig.Config, owner func(context.Context) error) *managedNodes { + if config.InstallationID == "" { + return nil } + result := &managedNodes{} result.hub = node.NewHub(node.HubOptions{ Generations: func(ctx context.Context, n node.Identity, connection string, epoch uint64, health node.Health) error { if err := owner(ctx); err != nil { @@ -96,41 +74,17 @@ func configureManagedNodes(nodes *deployment.Service, reader deployment.Reader, return nodes.Heartbeat(ctx, n.NodeID, connection, epoch, nodeHealthRecord(health)) }, }) - result.setup = &managedSetup{processPaths: providerProcessPaths(), registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: setupID, publicURL: publicURL} - result.runtime = execution.NewDeferredRuntimeProvider(setupID, result.setup.load, result.setup.prepare) + // Load requires OAC_PUBLIC_URL with an installation ID. + result.setup = &managedSetup{processPaths: config.ProviderPaths, registry: registry, deployment: nodes, allocations: reader, hub: result.hub, installationID: config.InstallationID, runtimeAPI: config.PublicOrigin.RuntimeAPI()} + result.runtime = execution.NewDeferredRuntimeProvider(config.InstallationID, result.setup.load, result.setup.prepare) result.runtime.PublishUnconfigured = result.setup.publishUnconfigured - success = true - return result, nil + return result } func (m *managedNodes) close() { if m != nil { m.hub.Close() - m.closeProvider() - } -} - -func deploymentAdminAuthenticator() (*api.DeploymentAuthenticator, error) { - path := os.Getenv("OAC_CORE_KEY_DIGESTS_FILE") - if path == "" { - return nil, nil - } - raw, err := os.ReadFile(path) - if err != nil { - return nil, errors.New("cannot read OAC_CORE_KEY_DIGESTS_FILE") - } - var digests []string - if json.Unmarshal(raw, &digests) != nil || len(digests) == 0 { - return nil, errors.New("OAC_CORE_KEY_DIGESTS_FILE must contain a JSON array of Core key SHA-256 digests") - } - return api.NewDeploymentAuthenticator(digests) -} - -func serverAddress() string { - if value := os.Getenv("OAC_ADDR"); value != "" { - return value } - return "127.0.0.1:8091" } func nodeHealthRecord(health node.Health) deployment.NodeHealth { diff --git a/services/core/cmd/server/managed_setup.go b/services/core/cmd/server/managed_setup.go index dfce8cdc6..0ca908728 100644 --- a/services/core/cmd/server/managed_setup.go +++ b/services/core/cmd/server/managed_setup.go @@ -29,8 +29,9 @@ type managedSetup struct { allocations generationAllocations hub *node.Hub installationID string - // publicURL is OAC_PUBLIC_URL; every sandbox reaches Core through it. - publicURL string + // runtimeAPI is the /api/v1 base of OAC_PUBLIC_URL; every sandbox reaches + // Core through it. + runtimeAPI string selected atomic.Pointer[managedSelection] providerCalls sandbox.CallFence } @@ -145,7 +146,7 @@ func (s *managedSetup) configuration(setup deployment.Setup) (execution.Prepared return execution.PreparedRuntimeDeployment{}, fmt.Errorf("%w: %v", execution.ErrExecutionUnavailable, err) } selected := &execution.RuntimeProvider{InstallationID: setup.InstallationID, ProviderKind: setup.Provider, Generation: setup.Generation, Mode: setup.Mode, AdmissionPaused: setup.AdmissionPaused, - CoreURL: s.publicURL + "/api/v1", BackendFingerprint: setup.BackendFingerprint, Provider: provider} + CoreURL: s.runtimeAPI, BackendFingerprint: setup.BackendFingerprint, Provider: provider} if setup.Suspension != nil { selected.Suspension = &execution.RuntimeSuspensionPolicy{IdleTimeout: time.Duration(setup.Suspension.IdleSeconds) * time.Second, Retention: time.Duration(setup.Suspension.RetentionSeconds) * time.Second, MaxActive: 4, MaxRetained: 16} diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index efc656985..d6d2ff663 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -2,17 +2,15 @@ package main import ( "context" - "crypto/sha256" - "encoding/hex" "errors" "os" "path/filepath" - "strings" "testing" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/deploymentpg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/docker" @@ -27,32 +25,18 @@ import ( ) func TestWebSetupCreatesManagerWithoutLocalProvider(t *testing.T) { - idFile := filepath.Join(t.TempDir(), "installation.id") - if err := os.WriteFile(idFile, []byte(uuid.NewString()+"\n"), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("OAC_INSTALLATION_ID_FILE", idFile) - digest := sha256.Sum256([]byte("synthetic-admin")) - path := filepath.Join(t.TempDir(), "core-key-digests.json") - if err := os.WriteFile(path, []byte(`["`+hex.EncodeToString(digest[:])+`"]`), 0600); err != nil { - t.Fatal(err) + if configureManagedNodes(nil, nil, providers.Builtin(), processconfig.Config{}, nil) != nil { + t.Fatal("sandbox manager started without an installation ID") } - t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", path) - if _, err := configureManagedNodes(nil, nil, providers.Builtin(), "", nil); err == nil || !strings.Contains(err.Error(), "OAC_PUBLIC_URL") { - t.Fatal("sandbox manager started without a public URL", err) - } - m, err := configureManagedNodes(nil, nil, providers.Builtin(), "https://core.example", func(context.Context) error { return nil }) + origin, err := deployment.NewPublicOrigin("https://core.example") if err != nil { t.Fatal(err) } + m := configureManagedNodes(nil, nil, providers.Builtin(), processconfig.Config{InstallationID: uuid.NewString(), PublicOrigin: &origin}, func(context.Context) error { return nil }) defer m.close() - if m.setup == nil || m.admin == nil || m.hub == nil || m.runtime == nil || m.runtime.Provider != nil { + if m.setup == nil || m.hub == nil || m.runtime == nil || m.runtime.Provider != nil || m.setup.runtimeAPI != "https://core.example/api/v1" { t.Fatal("zero-node setup unexpectedly instantiated local compute or omitted management") } - t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", "") - if _, err := configureManagedNodes(nil, nil, providers.Builtin(), "https://core.example", nil); err == nil { - t.Fatal("setup accepted without admin authentication") - } } // fakeDeploymentSetups is a strict deploymentSetups: a call without a set @@ -182,7 +166,7 @@ func TestManagedSetupPreparesWithoutPublishing(t *testing.T) { id := uuid.NewString() hub := node.NewHub(node.HubOptions{}) defer hub.Close() - s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, publicURL: "https://core.example"} + s := &managedSetup{registry: providers.Builtin(), installationID: id, hub: hub, deployment: &fakeDeploymentSetups{t: t}, allocations: &fakeGenerationAllocations{t: t}, runtimeAPI: "https://core.example/api/v1"} previous := &execution.RuntimeProvider{InstallationID: id, Generation: 1, ProviderKind: "docker"} s.publish(previous) candidate, err := s.prepare(t.Context(), deployment.Setup{InstallationID: id, Provider: "microsandbox", Mode: "nodes", Operations: microsandbox.Operations(), Suspension: &deployment.Suspension{IdleSeconds: 300, RetentionSeconds: 86400}}) diff --git a/services/core/cmd/server/oauth_refresh.go b/services/core/cmd/server/oauth_refresh.go deleted file mode 100644 index 8871bacd6..000000000 --- a/services/core/cmd/server/oauth_refresh.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import ( - "os" - "strings" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/oauthrefresh" -) - -func oauthRefreshClient() (*oauthrefresh.Client, error) { - var origins []string - if raw := os.Getenv("OAC_OAUTH_TRUSTED_ORIGINS"); raw != "" { - for _, origin := range strings.Split(raw, ",") { - origins = append(origins, strings.TrimSpace(origin)) - } - } - return oauthrefresh.NewClient(origins) -} diff --git a/services/core/cmd/server/oauth_refresh_test.go b/services/core/cmd/server/oauth_refresh_test.go deleted file mode 100644 index da8aaedf5..000000000 --- a/services/core/cmd/server/oauth_refresh_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package main - -import "testing" - -func TestOAuthRefreshOperatorPolicy(t *testing.T) { - for _, raw := range []string{"", "https://issuer.example", "https://issuer.example, https://10.0.0.1:9443"} { - t.Setenv("OAC_OAUTH_TRUSTED_ORIGINS", raw) - if _, err := oauthRefreshClient(); err != nil { - t.Fatal(err) - } - } - for _, raw := range []string{"http://issuer.example", "https://issuer.example/token", "https://issuer.example,", "https://user:secret@issuer.example"} { - t.Setenv("OAC_OAUTH_TRUSTED_ORIGINS", raw) - if _, err := oauthRefreshClient(); err == nil { - t.Fatal("invalid issuer policy accepted") - } - } -} diff --git a/services/core/cmd/server/process_configuration.go b/services/core/cmd/server/process_configuration.go deleted file mode 100644 index 476973ac5..000000000 --- a/services/core/cmd/server/process_configuration.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "os" - - "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" -) - -// The launcher loads core.env. Core reports its sources without parsing another -// configuration layer or logging environment values. -func logConfigurationSources() { - log.Bg().Info("Core process configuration loaded from the process environment") - for _, key := range []string{"OAC_HISTORY_SETTINGS_FILE"} { - if path := os.Getenv(key); path != "" { - log.Bg().Info("Core auxiliary configuration", "setting", key, "path", path) - } - } -} - -func providerProcessPaths() sandbox.ProcessPaths { - return sandbox.ProcessPaths{ArtifactRoot: os.Getenv("OAC_PROVIDER_ROOT"), StateRoot: os.Getenv("OAC_PROVIDER_STATE_ROOT")} -} diff --git a/services/core/cmd/server/process_configuration_test.go b/services/core/cmd/server/process_configuration_test.go deleted file mode 100644 index eef02edc8..000000000 --- a/services/core/cmd/server/process_configuration_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "os" - "strings" - "testing" - - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" -) - -func TestExecutionConcurrencyConfiguration(t *testing.T) { - t.Setenv("OAC_EXECUTION_CONCURRENCY", "unused") - if err := os.Unsetenv("OAC_EXECUTION_CONCURRENCY"); err != nil { - t.Fatal(err) - } - if got, err := processconfig.ExecutionConcurrency(); err != nil || got != 4 { - t.Fatal(got, err) - } - t.Setenv("OAC_EXECUTION_CONCURRENCY", "") - if got, err := processconfig.ExecutionConcurrency(); err != nil || got != 4 { - t.Fatal("empty concurrency did not keep the default", got, err) - } - for _, value := range []string{"1", "7", "1024"} { - t.Setenv("OAC_EXECUTION_CONCURRENCY", value) - if got, err := processconfig.ExecutionConcurrency(); err != nil || got < 1 { - t.Fatal(value, got, err) - } - } - for _, value := range []string{"0", "-1", "1025", "1.5", "secret-value"} { - t.Setenv("OAC_EXECUTION_CONCURRENCY", value) - if _, err := processconfig.ExecutionConcurrency(); err == nil || strings.Contains(err.Error(), "secret-value") { - t.Fatal("invalid concurrency accepted or echoed", err) - } - } -} - -func TestPublicURLMustBeACanonicalOrigin(t *testing.T) { - for _, value := range []string{"https://core.example", "https://core.example:8443", "http://127.0.0.1:8091", "http://core.example"} { - t.Setenv("OAC_PUBLIC_URL", value) - if got, err := processconfig.PublicURL(); err != nil || got != value { - t.Fatal(value, got, err) - } - } - for _, value := range []string{"https://core.example/", "https://Core.example", "wss://core.example", "https://core.example/v1"} { - t.Setenv("OAC_PUBLIC_URL", value) - if _, err := processconfig.PublicURL(); err == nil { - t.Fatal("accepted", value) - } - } -} diff --git a/services/core/cmd/server/runtime_history.go b/services/core/cmd/server/runtime_history.go index 11213fa2e..7d333c097 100644 --- a/services/core/cmd/server/runtime_history.go +++ b/services/core/cmd/server/runtime_history.go @@ -1,44 +1,17 @@ package main import ( - "bytes" "context" - "encoding/json" - "errors" - "io" - "net/url" - "os" - "strings" "time" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/runtimehistorypg" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimehistory" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs/otlpexporter" - "golang.org/x/net/http/httpguts" ) -const ( - defaultRuntimeHistoryQueueCapacity = 256 - defaultRuntimeHistoryTimeoutSeconds = 2 - maxRuntimeHistoryQueueCapacity = 4096 - maxRuntimeHistoryTimeoutSeconds = 30 - minRuntimeHistorySampleIntervalSeconds = 5 - maxRuntimeHistorySampleIntervalSeconds = 300 -) - -type runtimeHistoryConfig struct { - Transport string `json:"transport,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - Insecure bool `json:"insecure,omitempty"` - Headers map[string]string `json:"headers,omitempty"` - QueueCapacity int `json:"queue_capacity,omitempty"` - TimeoutSeconds int `json:"timeout_seconds,omitempty"` - SampleIntervalSeconds int `json:"sample_interval_seconds,omitempty"` -} - type runtimeHistorySetup struct { Options []runtimeobs.ServiceOption Exporter runtimeHistoryExporter @@ -52,12 +25,8 @@ type runtimeHistoryExporter interface { Close(context.Context) error } -func runtimeHistory(ctx context.Context, units *pgunit.Pool, executionEnabled bool) (runtimeHistorySetup, error) { - config, err := loadRuntimeHistoryConfig() - if err != nil { - return runtimeHistorySetup{}, err - } - interval := time.Duration(config.SampleIntervalSeconds) * time.Second +func runtimeHistory(ctx context.Context, units *pgunit.Pool, config processconfig.RuntimeHistory, executionEnabled bool) (runtimeHistorySetup, error) { + interval := config.SampleInterval mode := runtimehistory.CollectionPeriodic if !executionEnabled { interval = 0 @@ -69,15 +38,14 @@ func runtimeHistory(ctx context.Context, units *pgunit.Pool, executionEnabled bo MaximumPoints: 1000, MaximumSeries: 64, MaximumTotalPoints: 10000, Metrics: []runtimehistory.Metric{runtimehistory.MetricCPU, runtimehistory.MetricMemory, runtimehistory.MetricTokens}, } - timeout := time.Duration(config.TimeoutSeconds) * time.Second - backend, err := runtimehistorypg.New(units, runtimehistorypg.Config{Capabilities: capabilities, QueryTimeout: timeout}) + backend, err := runtimehistorypg.New(units, runtimehistorypg.Config{Capabilities: capabilities, QueryTimeout: config.Timeout}) if err != nil { return runtimeHistorySetup{}, err } - options := runtimeobs.ExportOptions{QueueCapacity: config.QueueCapacity, Timeout: timeout} + options := runtimeobs.ExportOptions{QueueCapacity: config.QueueCapacity, Timeout: config.Timeout} setup := runtimeHistorySetup{Options: []runtimeobs.ServiceOption{runtimeobs.WithExporter(backend, options)}, Reader: backend, SampleInterval: interval, Prune: backend.Prune} if config.Endpoint != "" { - exporter, err := otlpexporter.New(ctx, otlpexporter.Config{Endpoint: config.Endpoint, Headers: config.Headers, Insecure: config.Insecure, RequestTimeout: timeout}) + exporter, err := otlpexporter.New(ctx, otlpexporter.Config{Endpoint: config.Endpoint, Headers: config.Headers, Insecure: config.Insecure, RequestTimeout: config.Timeout}) if err != nil { return runtimeHistorySetup{}, err } @@ -88,99 +56,7 @@ func runtimeHistory(ctx context.Context, units *pgunit.Pool, executionEnabled bo return setup, nil } -func loadRuntimeHistoryConfig() (runtimeHistoryConfig, error) { - var config runtimeHistoryConfig - if file := os.Getenv("OAC_HISTORY_SETTINGS_FILE"); file != "" { - raw, err := os.ReadFile(file) - if err != nil { - return config, errors.New("cannot read OAC_HISTORY_SETTINGS_FILE") - } - decoder := json.NewDecoder(bytes.NewReader(raw)) - decoder.DisallowUnknownFields() - if decoder.Decode(&config) != nil || decoder.Decode(new(any)) != io.EOF { - return config, errors.New("invalid Runtime history configuration") - } - } - if err := validateRuntimeHistoryConfig(config); err != nil { - return config, err - } - if config.QueueCapacity == 0 { - config.QueueCapacity = defaultRuntimeHistoryQueueCapacity - } - if config.TimeoutSeconds == 0 { - config.TimeoutSeconds = defaultRuntimeHistoryTimeoutSeconds - } - if config.SampleIntervalSeconds == 0 { - config.SampleIntervalSeconds = 30 - } - return config, nil -} - func closeRuntimeHistory(ctx context.Context, exporter runtimeHistoryExporter) { defer func() { _ = recover() }() _ = exporter.Close(ctx) } - -// Retention also runs without active Runtimes. Each bounded pass has its own deadline. -func runHistoryCleanup(ctx context.Context, prune func(context.Context) (int64, error), metrics *coremetrics.Service) { - if metrics != nil { - defer metrics.StopJob("history_cleanup") - } - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - pruneCtx, cancel := context.WithTimeout(ctx, 2*time.Second) - count, err := prune(pruneCtx) - cancel() - reportCleanupResult(metrics, "history_cleanup", count, err) - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - } -} - -func validateRuntimeHistoryConfig(config runtimeHistoryConfig) error { - if config.QueueCapacity < 0 || config.QueueCapacity > maxRuntimeHistoryQueueCapacity { - return errors.New("Runtime history queue_capacity is out of range") - } - if config.TimeoutSeconds < 0 || config.TimeoutSeconds > maxRuntimeHistoryTimeoutSeconds { - return errors.New("Runtime history timeout_seconds is out of range") - } - if config.SampleIntervalSeconds != 0 && (config.SampleIntervalSeconds < minRuntimeHistorySampleIntervalSeconds || config.SampleIntervalSeconds > maxRuntimeHistorySampleIntervalSeconds) { - return errors.New("Runtime history sample_interval_seconds is out of range") - } - if config.Endpoint == "" { - if config.Transport != "" || config.Insecure || len(config.Headers) != 0 { - return errors.New("Runtime history export options require an endpoint") - } - return nil - } - if config.Transport != "otlp_http" { - return errors.New("Runtime history transport must be otlp_http") - } - endpoint, err := url.Parse(config.Endpoint) - if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" || endpoint.RawPath != "" || endpoint.Path == "" || endpoint.String() != config.Endpoint { - return errors.New("Runtime history endpoint must be a canonical absolute OTLP metrics URL") - } - switch endpoint.Scheme { - case "https": - if config.Insecure { - return errors.New("Runtime history insecure transport requires an http endpoint") - } - case "http": - if !config.Insecure { - return errors.New("Runtime history http endpoint requires insecure=true") - } - default: - return errors.New("Runtime history endpoint scheme must be https or explicit insecure http") - } - for key, value := range config.Headers { - lower := strings.ToLower(key) - if !httpguts.ValidHeaderFieldName(key) || !httpguts.ValidHeaderFieldValue(value) || lower == "host" || lower == "content-length" || lower == "content-type" || lower == "content-encoding" { - return errors.New("Runtime history headers contain an invalid or reserved entry") - } - } - return nil -} diff --git a/services/core/cmd/server/runtime_history_test.go b/services/core/cmd/server/runtime_history_test.go index c640f2c92..17c748cb5 100644 --- a/services/core/cmd/server/runtime_history_test.go +++ b/services/core/cmd/server/runtime_history_test.go @@ -2,13 +2,12 @@ package main import ( "context" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" - "os" - "path/filepath" - "strings" "testing" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgunit" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/processconfig" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimeobs" ) type panicHistoryExporter struct{} @@ -16,10 +15,11 @@ type panicHistoryExporter struct{} func (panicHistoryExporter) Export(context.Context, runtimeobs.ExportRecord) error { return nil } func (panicHistoryExporter) Close(context.Context) error { panic("close") } +var defaultHistory = processconfig.RuntimeHistory{QueueCapacity: 256, Timeout: 2 * time.Second, SampleInterval: 30 * time.Second} + func TestRuntimeHistoryUsesCoreDatabaseByDefault(t *testing.T) { - t.Setenv("OAC_HISTORY_SETTINGS_FILE", "") for _, enabled := range []bool{true, false} { - setup, err := runtimeHistory(t.Context(), pgunit.NewPool(nil), enabled) + setup, err := runtimeHistory(t.Context(), pgunit.NewPool(nil), defaultHistory, enabled) if err != nil { t.Fatal(err) } @@ -40,12 +40,9 @@ func TestRuntimeHistoryUsesCoreDatabaseByDefault(t *testing.T) { } func TestRuntimeHistoryOptionalExportAndSamplingConfiguration(t *testing.T) { - file := filepath.Join(t.TempDir(), "history.json") - if err := os.WriteFile(file, []byte(`{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","headers":{"Authorization":"Bearer private"},"sample_interval_seconds":60}`), 0600); err != nil { - t.Fatal(err) - } - t.Setenv("OAC_HISTORY_SETTINGS_FILE", file) - setup, err := runtimeHistory(t.Context(), pgunit.NewPool(nil), true) + config := defaultHistory + config.Endpoint, config.Headers, config.SampleInterval = "https://collector.example.test/v1/metrics", map[string]string{"Authorization": "Bearer private"}, time.Minute + setup, err := runtimeHistory(t.Context(), pgunit.NewPool(nil), config, true) if err != nil { t.Fatal(err) } @@ -55,51 +52,6 @@ func TestRuntimeHistoryOptionalExportAndSamplingConfiguration(t *testing.T) { } } -func TestRuntimeHistoryConfigFailsClosedWithoutLeakingSecrets(t *testing.T) { - tests := []struct { - name string - config string - }{ - {name: "unknown field", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","secret":"must-not-leak"}`}, - {name: "implicit insecure", config: `{"transport":"otlp_http","endpoint":"http://collector.example.test/v1/metrics"}`}, - {name: "userinfo", config: `{"transport":"otlp_http","endpoint":"https://user:must-not-leak@collector.example.test/v1/metrics"}`}, - {name: "header newline", config: "{\"transport\":\"otlp_http\",\"endpoint\":\"https://collector.example.test/v1/metrics\",\"headers\":{\"Authorization\":\"Bearer must-not-leak\\n\"}}"}, - {name: "reserved header", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","headers":{"Host":"must-not-leak"}}`}, - {name: "oversized queue", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","queue_capacity":4097}`}, - {name: "oversized timeout", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","timeout_seconds":31}`}, - {name: "too frequent sampling", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","sample_interval_seconds":4}`}, - {name: "oversized sampling interval", config: `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","sample_interval_seconds":301}`}, - {name: "removed backend", config: `{"clickhouse":{"password":"must-not-leak"}}`}, - {name: "transport without endpoint", config: `{"transport":"otlp_http"}`}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - file := filepath.Join(t.TempDir(), "runtime-history.json") - if err := os.WriteFile(file, []byte(test.config), 0600); err != nil { - t.Fatal(err) - } - t.Setenv("OAC_HISTORY_SETTINGS_FILE", file) - _, err := loadRuntimeHistoryConfig() - if err == nil { - t.Fatal("unsafe history configuration accepted") - } - if strings.Contains(err.Error(), "must-not-leak") { - t.Fatalf("history error leaked config content: %v", err) - } - }) - } -} - -func TestRuntimeHistoryAllowsExplicitLocalHTTPCollector(t *testing.T) { - config := runtimeHistoryConfig{ - Transport: "otlp_http", Endpoint: "http://127.0.0.1:4318/v1/metrics", Insecure: true, - Headers: map[string]string{"X-Scope-OrgID": "operator-history"}, - } - if err := validateRuntimeHistoryConfig(config); err != nil { - t.Fatal(err) - } -} - func TestRuntimeHistoryClosePanicIsIsolated(t *testing.T) { closeRuntimeHistory(t.Context(), panicHistoryExporter{}) } diff --git a/services/core/cmd/server/write_audit.go b/services/core/cmd/server/write_audit.go deleted file mode 100644 index 481533521..000000000 --- a/services/core/cmd/server/write_audit.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "context" - "errors" - "os" - "time" - - "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" -) - -type writeAuditPruner interface { - DeleteExpiredWriteOperations(context.Context, time.Time, int) (int64, error) -} - -func writeAuditRetention() (time.Duration, error) { - value := os.Getenv("OAC_WRITE_AUDIT_RETENTION") - if value == "" { - return 90 * 24 * time.Hour, nil - } - duration, err := time.ParseDuration(value) - if err != nil || duration < time.Hour { - return 0, errors.New("OAC_WRITE_AUDIT_RETENTION must be a duration of at least 1h") - } - return duration, nil -} - -func runWriteAuditCleanup(ctx context.Context, s writeAuditPruner, retention time.Duration, metrics *coremetrics.Service) { - if metrics != nil { - defer metrics.StopJob("audit_cleanup") - } - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - pruneCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - count, err := s.DeleteExpiredWriteOperations(pruneCtx, time.Now().Add(-retention), 1000) - cancel() - reportCleanupResult(metrics, "audit_cleanup", count, err) - if err != nil && ctx.Err() == nil { - log.Ctx(ctx).Warn("Write audit retention cleanup failed") - } - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - } -} diff --git a/services/core/cmd/server/write_audit_test.go b/services/core/cmd/server/write_audit_test.go deleted file mode 100644 index 3891684c1..000000000 --- a/services/core/cmd/server/write_audit_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "context" - "errors" - "testing" - "time" -) - -func TestWriteAuditRetention(t *testing.T) { - for _, test := range []struct { - value string - want time.Duration - bad bool - }{{"", 90 * 24 * time.Hour, false}, {"24h", 24 * time.Hour, false}, {"0", 0, true}, {"30m", 0, true}, {"-1h", 0, true}, {"90d", 0, true}} { - t.Run(test.value, func(t *testing.T) { - t.Setenv("OAC_WRITE_AUDIT_RETENTION", test.value) - got, err := writeAuditRetention() - if (err != nil) != test.bad || (!test.bad && got != test.want) { - t.Fatalf("%v %v", got, err) - } - }) - } -} - -type auditPruneProbe struct { - cancel context.CancelFunc - called bool - cutoff time.Time - limit int - deadline bool -} - -func (p *auditPruneProbe) DeleteExpiredWriteOperations(ctx context.Context, cutoff time.Time, limit int) (int64, error) { - p.called = true - p.cutoff = cutoff - p.limit = limit - _, p.deadline = ctx.Deadline() - p.cancel() - return 0, errors.New("test") -} -func TestWriteAuditCleanupBoundedAndCancellable(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - probe := &auditPruneProbe{cancel: cancel} - before := time.Now().Add(-24 * time.Hour) - runWriteAuditCleanup(ctx, probe, 24*time.Hour, nil) - if !probe.called || probe.limit != 1000 || !probe.deadline || probe.cutoff.Before(before) || probe.cutoff.After(time.Now().Add(-24*time.Hour)) { - t.Fatalf("bad cleanup %+v", probe) - } -} diff --git a/services/core/deploy/claude/Dockerfile b/services/core/deploy/claude/Dockerfile index 08ffc4eee..115878e19 100644 --- a/services/core/deploy/claude/Dockerfile +++ b/services/core/deploy/claude/Dockerfile @@ -10,7 +10,6 @@ COPY claude-sdk /opt/claude-sdk ENV HOME=/home/runtime OAC_RUNTIME_HOME=/home/runtime/.oac \ OAC_RUNTIME_CLAUDE_SDK_NODE=/usr/local/bin/node \ OAC_RUNTIME_CLAUDE_SDK_ENTRYPOINT=/opt/claude-sdk/dist/main.js \ - OAC_RUNTIME_CLAUDE_SDK_WORKSPACE=managed \ OAC_RUNTIME_WORKSPACE=/environment/workspace \ OAC_RUNTIME_INITIALIZATION_DIRECTORY=/environment/initialization \ OAC_RUNTIME_PACKAGE_DIRECTORY=/environment/packages diff --git a/services/core/internal/api/contract_routes_test.go b/services/core/internal/api/contract_routes_test.go index 9d4fadb95..6b589bceb 100644 --- a/services/core/internal/api/contract_routes_test.go +++ b/services/core/internal/api/contract_routes_test.go @@ -69,7 +69,7 @@ func TestContractsPublishExactlyTheRegisteredCoreAndMachineRoutes(t *testing.T) // Every optional group that gates a route registration, as the server enables them. deps, fakes := testDependencies(t) deps.Execution, deps.Sandboxes = fakes.execution(), fakes.sandboxes() - deps.Execution.NativeInstaller = &NativeInstaller{Version: "contract-test", Catalog: &nativeinstaller.Catalog{}} + deps.Execution.NativeInstaller = &NativeInstaller{Version: "contract-test", Base: "https://core.example/api/v1/agent-daemon/install/", Catalog: &nativeinstaller.Catalog{}} h := &Handler{Dependencies: deps} contracts := map[string]string{"/v1": "openapi.yaml", "/core/v1": "core.openapi.yaml", "/api/v1": "runtime.openapi.yaml"} published := map[string]map[string]bool{} diff --git a/services/core/internal/api/dependencies.go b/services/core/internal/api/dependencies.go index f4e2b408d..5dcb4f981 100644 --- a/services/core/internal/api/dependencies.go +++ b/services/core/internal/api/dependencies.go @@ -151,8 +151,8 @@ func (d Dependencies) validate() error { if e.ExecutorURL == "" { return errors.New("api: Execution.ExecutorURL is required") } - if e.NativeInstaller != nil && e.NativeInstaller.Version == "" { - return errors.New("api: Execution.NativeInstaller.Version is required") + if e.NativeInstaller != nil && (e.NativeInstaller.Version == "" || e.NativeInstaller.Base == "") { + return errors.New("api: Execution.NativeInstaller.Version and Base are required") } if err := required( field{"Execution.SessionAdmission", e.SessionAdmission}, diff --git a/services/core/internal/api/dependencies_test.go b/services/core/internal/api/dependencies_test.go index c7a5bf73b..a044d4988 100644 --- a/services/core/internal/api/dependencies_test.go +++ b/services/core/internal/api/dependencies_test.go @@ -179,7 +179,7 @@ func TestNewHandlerAcceptsCompleteDependencies(t *testing.T) { t.Fatal(err) } deps.Execution = f.execution() - deps.Execution.NativeInstaller = &NativeInstaller{Version: "build"} + deps.Execution.NativeInstaller = &NativeInstaller{Version: "build", Base: "https://core.example/api/v1/agent-daemon/install/"} deps.Sandboxes = f.sandboxes() if _, err := NewHandler(deps); err != nil { t.Fatal(err) diff --git a/services/core/internal/api/environment_installation.go b/services/core/internal/api/environment_installation.go index 8f60a15c9..caa666ef7 100644 --- a/services/core/internal/api/environment_installation.go +++ b/services/core/internal/api/environment_installation.go @@ -17,6 +17,8 @@ import ( type NativeInstaller struct { // Version is the build revision executors install and claim. Version string + // Base is the public URL prefix of the versioned installer downloads. + Base string // Catalog holds the matching installation artifacts. It is nil when the // operator installed none: installations then report unavailable and the // grant routes answer 503 installation_unavailable. @@ -46,9 +48,7 @@ func (h *Handler) installationFor(ctx context.Context, principal identity.Princi if err != nil { return nil, err } - origin := strings.TrimSuffix(h.Execution.ExecutorURL, "/api/v1/agent-daemon/ws") - origin = strings.Replace(strings.Replace(origin, "wss://", "https://", 1), "ws://", "http://", 1) - return &v1.EnvironmentInstallation{Status: "available", Version: installer.Version, ExpiresAt: expires, Commands: installer.Catalog.Commands(origin, token)}, nil + return &v1.EnvironmentInstallation{Status: "available", Version: installer.Version, ExpiresAt: expires, Commands: installer.Catalog.Commands(installer.Base, token)}, nil } func (h *Handler) addSessionInstallation(w http.ResponseWriter, r *http.Request, response *v1.Session) error { diff --git a/services/core/internal/api/environment_installation_test.go b/services/core/internal/api/environment_installation_test.go index 828342045..4cf35ef2d 100644 --- a/services/core/internal/api/environment_installation_test.go +++ b/services/core/internal/api/environment_installation_test.go @@ -40,7 +40,7 @@ func TestSelfHostedCreationReturnsInstallationWithoutWebCredential(t *testing.T) fakes.modelProviders.resolve = fixtureDeploymentProvider fakes.environments.authorizeEnvironmentInstallation, fakes.environments.validateEnvironmentInstallation = f.AuthorizeEnvironmentInstallation, f.ValidateEnvironmentInstallation deps.Execution = fakes.execution() - deps.Execution.NativeInstaller = &NativeInstaller{Version: "build", Catalog: &nativeinstaller.Catalog{Version: "build"}} + deps.Execution.NativeInstaller = &NativeInstaller{Version: "build", Base: "https://core.example/api/v1/agent-daemon/install/", Catalog: &nativeinstaller.Catalog{Version: "build"}} handler := newTestHandler(t, deps) body := `{"agent":{"model":"model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"},"x_agents_core":{"model_provider":{"protocol":"responses","base_url":"https://model.example/v1","api_key":"fixture-model"}}}` r := httptest.NewRequest(http.MethodPost, "/v1/agents/sessions", strings.NewReader(body)) diff --git a/services/core/internal/api/project_api_key_configuration.go b/services/core/internal/api/project_api_key_configuration.go index 19b88da7c..1467610ec 100644 --- a/services/core/internal/api/project_api_key_configuration.go +++ b/services/core/internal/api/project_api_key_configuration.go @@ -14,9 +14,6 @@ type projectKeyDigests interface { // ValidateCredentialSeparation rejects Core key collisions with persisted API keys. func ValidateCredentialSeparation(ctx context.Context, admin *DeploymentAuthenticator, keys projectKeyDigests) error { - if admin == nil { - return errors.New("OAC_CORE_KEY_DIGESTS_FILE is required; Core needs the Core key digest") - } for digest := range admin.digests { exists, err := keys.APIKeyDigestExists(ctx, digest) if err != nil { diff --git a/services/core/internal/api/project_api_keys_test.go b/services/core/internal/api/project_api_keys_test.go index a7e2e42bc..9847d0770 100644 --- a/services/core/internal/api/project_api_keys_test.go +++ b/services/core/internal/api/project_api_keys_test.go @@ -96,9 +96,6 @@ func (s *separationFixture) APIKeyDigestExists(_ context.Context, digest [sha256 } func TestAdministratorCredentialSeparation(t *testing.T) { s := &separationFixture{} - if err := ValidateCredentialSeparation(t.Context(), nil, s); err == nil { - t.Fatal("missing administrator accepted") - } admin, _ := NewDeploymentAuthenticator([]string{runtimedevice.HashCredential("admin")}) if err := ValidateCredentialSeparation(t.Context(), admin, s); err != nil || len(s.checked) != 1 || s.checked[0] != sha256.Sum256([]byte("admin")) { t.Fatal("administrator digest was not checked against persisted keys", err) diff --git a/services/core/internal/coremetrics/service.go b/services/core/internal/coremetrics/service.go index b5433c329..cfb24849d 100644 --- a/services/core/internal/coremetrics/service.go +++ b/services/core/internal/coremetrics/service.go @@ -2,10 +2,13 @@ package coremetrics import ( "context" + "errors" "regexp" "runtime" "sync" "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) const SampleInterval = 30 * time.Second @@ -14,7 +17,6 @@ const retention = 7 * 24 * time.Hour // The 7d window ends at a complete 2h bucket, so retain its leading padding too. const sampleCapacity = int((retention+2*time.Hour)/SampleInterval) + 2 -var jobIDs = []string{"scheduler", "runtime_sampler", "history_cleanup", "audit_cleanup"} var revisionPattern = regexp.MustCompile(`^[0-9a-f]{40}$`) type refusalSlot struct { @@ -31,19 +33,43 @@ type Service struct { nextSample int refusals [sampleCapacity]refusalSlot latest Sample + jobIDs []string jobs map[string]Job + periodic []Periodic process processSampler } -func New(started time.Time, revision string, source Source) *Service { - s := &Service{source: source, started: started.UTC(), now: time.Now, jobs: map[string]Job{}} +// Periodic is a background job the metrics report. Run makes one bounded pass +// and returns what it processed, nil when the pass counted nothing, and what +// failed. The next pass starts Every after a pass ends; a panic fails that pass +// only. A job without Run is disabled and reports stopped. +type Periodic struct { + ID string + Every time.Duration + Run func(context.Context) (processed *int64, failed int64, err error) +} + +// errPanicked is the error of a pass that panicked. +var errPanicked = errors.New("periodic job pass panicked") + +// New reports the scheduler, which the Source reads live, and then jobs in +// their order. Run runs the jobs. An enabled job needs a positive Every. +func New(started time.Time, revision string, source Source, jobs ...Periodic) (*Service, error) { + s := &Service{source: source, started: started.UTC(), now: time.Now, jobIDs: []string{"scheduler"}, jobs: map[string]Job{"scheduler": {ID: "scheduler", Status: "unknown"}}, periodic: jobs} if revisionPattern.MatchString(revision) { s.revision = &revision } - for _, id := range jobIDs { - s.jobs[id] = Job{ID: id, Status: "unknown"} + for _, job := range jobs { + status := "unknown" + if job.Run == nil { + status = "stopped" + } else if job.Every <= 0 { + return nil, errors.New("coremetrics: periodic job " + job.ID + " needs a positive interval") + } + s.jobIDs = append(s.jobIDs, job.ID) + s.jobs[job.ID] = Job{ID: job.ID, Status: status} } - return s + return s, nil } func slot(t time.Time) (int64, int) { tick := t.Unix() / int64(SampleInterval/time.Second) @@ -61,7 +87,7 @@ func (s *Service) RecordUnavailable() { } s.refusals[i].count++ } -func (s *Service) ReportJob(id string, at time.Time, processed *int64, failed *int64, err error) { +func (s *Service) reportJob(id string, at time.Time, processed *int64, failed *int64, err error) { status := "ok" if err != nil || (failed != nil && *failed > 0) { status = "failing" @@ -73,7 +99,7 @@ func (s *Service) ReportJob(id string, at time.Time, processed *int64, failed *i } s.jobs[id] = Job{ID: id, Status: status, LastRunAt: ptr(at.UTC()), Processed: processed, Failed: failed} } -func (s *Service) StopJob(id string) { +func (s *Service) stopJob(id string) { s.mu.Lock() defer s.mu.Unlock() if j, ok := s.jobs[id]; ok { @@ -81,7 +107,46 @@ func (s *Service) StopJob(id string) { s.jobs[id] = j } } + +// Run samples Core and runs every enabled job until ctx ends, then waits for +// them to stop. func (s *Service) Run(ctx context.Context) { + var jobs sync.WaitGroup + for _, job := range s.periodic { + if job.Run != nil { + jobs.Go(func() { s.runJob(ctx, job) }) + } + } + s.sample(ctx) + jobs.Wait() +} + +func (s *Service) runJob(ctx context.Context, job Periodic) { + defer s.stopJob(job.ID) + for { + s.pass(ctx, job) + select { + case <-ctx.Done(): + return + case <-time.After(job.Every): + } + } +} + +// pass runs and reports one pass of job. A panic is reported as a failed pass. +func (s *Service) pass(ctx context.Context, job Periodic) { + var processed *int64 + failed, err := int64(1), errPanicked + defer func() { + if recovered := recover(); recovered != nil { + log.Ctx(ctx).Error("Periodic job panicked", "job", job.ID, "panic", recovered) + } + s.reportJob(job.ID, s.now(), processed, &failed, err) + }() + processed, failed, err = job.Run(ctx) +} + +func (s *Service) sample(ctx context.Context) { ticker := time.NewTicker(SampleInterval) defer ticker.Stop() for { @@ -132,7 +197,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { } live := s.source.Live() view := View{Object: "core.metrics", Range: window, Service: ServiceState{Status: "running", Revision: s.revision, StartedAt: ptr(s.started), ExecutionOwner: live.ExecutionOwner}, - Execution: Execution{SlotsInUse: live.SlotsInUse, SlotsTotal: live.SlotsTotal, ConnectedDaemons: live.ConnectedDaemons}, Database: Database{Pool: live.Pool}, Jobs: make([]Job, 0, len(jobIDs))} + Execution: Execution{SlotsInUse: live.SlotsInUse, SlotsTotal: live.SlotsTotal, ConnectedDaemons: live.ConnectedDaemons}, Database: Database{Pool: live.Pool}, Jobs: make([]Job, 0, len(s.jobIDs))} s.mu.Lock() latest := s.latest if latest.At.IsZero() || now.Sub(latest.At) > 2*SampleInterval || !latest.Healthy { @@ -145,7 +210,7 @@ func (s *Service) Read(ctx context.Context, name string) (View, error) { view.Process = latest.Process } - for _, id := range jobIDs { + for _, id := range s.jobIDs { j := s.jobs[id] if id == "scheduler" && live.Scheduler.ID != "" { j = live.Scheduler diff --git a/services/core/internal/coremetrics/service_test.go b/services/core/internal/coremetrics/service_test.go index 44a3ab294..d582a2827 100644 --- a/services/core/internal/coremetrics/service_test.go +++ b/services/core/internal/coremetrics/service_test.go @@ -26,7 +26,10 @@ func fixtureService(t *testing.T) (*Service, *fixtureSource, time.Time) { t.Helper() now := time.Date(2026, 9, 25, 12, 0, 20, 0, time.UTC) source := &fixtureSource{history: History{Buckets: map[time.Time]*float64{}}, live: Live{ExecutionOwner: ptr(true), SlotsInUse: ptr(int64(2)), SlotsTotal: ptr(int64(4))}} - service := New(now.Add(-2*time.Hour), strings.Repeat("a", 40), source) + service, err := New(now.Add(-2*time.Hour), strings.Repeat("a", 40), source, Periodic{ID: "runtime_sampler"}, Periodic{ID: "history_cleanup"}, Periodic{ID: "audit_cleanup"}) + if err != nil { + t.Fatal(err) + } service.now = func() time.Time { return now } return service, source, now } @@ -121,17 +124,17 @@ func TestAllRangesAndBoundedRetention(t *testing.T) { func TestJobResultsAndConcurrentReads(t *testing.T) { s, _, now := fixtureService(t) s.record(Sample{At: now, Healthy: true}) - s.ReportJob("audit_cleanup", now, ptr(int64(12)), ptr(int64(0)), nil) + s.reportJob("audit_cleanup", now, ptr(int64(12)), ptr(int64(0)), nil) got, _ := s.Read(t.Context(), "1h") if got.Service.Status != "running" || *got.Jobs[3].Processed != 12 { t.Fatal(got.Service, got.Jobs) } - s.ReportJob("audit_cleanup", now, ptr(int64(2)), ptr(int64(1)), errors.New("failure")) + s.reportJob("audit_cleanup", now, ptr(int64(2)), ptr(int64(1)), errors.New("failure")) got, _ = s.Read(t.Context(), "1h") if got.Service.Status != "degraded" { t.Fatal(got.Service) } - s.StopJob("audit_cleanup") + s.stopJob("audit_cleanup") got, _ = s.Read(t.Context(), "1h") if got.Jobs[3].Status != "stopped" { t.Fatal(got.Jobs) @@ -143,7 +146,7 @@ func TestJobResultsAndConcurrentReads(t *testing.T) { defer wg.Done() for range 20 { s.RecordUnavailable() - s.ReportJob("history_cleanup", now, ptr(int64(0)), ptr(int64(0)), nil) + s.reportJob("history_cleanup", now, ptr(int64(0)), ptr(int64(0)), nil) if _, err := s.Read(context.Background(), "1h"); err != nil { t.Error(err) } @@ -152,10 +155,48 @@ func TestJobResultsAndConcurrentReads(t *testing.T) { } wg.Wait() } +func TestPeriodicJobsRunUntilStopped(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + run := func(context.Context) (*int64, int64, error) { + cancel() + return nil, 1, errors.New("failure") + } + if _, err := New(time.Now(), "", &fixtureSource{}, Periodic{ID: "audit_cleanup", Run: run}); err == nil { + t.Fatal("accepted a job without an interval") + } + s, err := New(time.Now(), "", &fixtureSource{}, Periodic{ID: "runtime_sampler"}, Periodic{ID: "audit_cleanup", Every: time.Hour, Run: run}) + if err != nil { + t.Fatal(err) + } + s.Run(ctx) + if strings.Join(s.jobIDs, ",") != "scheduler,runtime_sampler,audit_cleanup" || s.jobs["runtime_sampler"].Status != "stopped" { + t.Fatal(s.jobIDs, s.jobs) + } + if job := s.jobs["audit_cleanup"]; job.Status != "stopped" || job.LastRunAt == nil || job.Processed != nil || *job.Failed != 1 { + t.Fatal(job) + } +} +func TestPeriodicJobSurvivesAPanic(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + passes := 0 + s, err := New(time.Now(), "", &fixtureSource{}, Periodic{ID: "audit_cleanup", Every: time.Millisecond, Run: func(context.Context) (*int64, int64, error) { + if passes++; passes == 1 { + panic("test") + } + cancel() + return ptr(int64(3)), 0, nil + }}) + if err != nil { + t.Fatal(err) + } + s.Run(ctx) + if job := s.jobs["audit_cleanup"]; passes != 2 || job.Processed == nil || *job.Processed != 3 || *job.Failed != 0 { + t.Fatal(passes, job) + } +} func TestRevisionMustBeCommit(t *testing.T) { for _, revision := range []string{"", "unknown", "secret-value"} { - s := New(time.Now(), revision, &fixtureSource{}) - if s.revision != nil { + if s, _ := New(time.Now(), revision, &fixtureSource{}); s.revision != nil { t.Fatal("unverified build revision exposed") } } diff --git a/services/core/internal/deployment/public_url.go b/services/core/internal/deployment/public_url.go index 999bcd558..1bf556928 100644 --- a/services/core/internal/deployment/public_url.go +++ b/services/core/internal/deployment/public_url.go @@ -8,45 +8,66 @@ import ( "strings" ) -// ValidateCoreURL accepts a canonical public origin, never a path or +// PublicOrigin is OAC_PUBLIC_URL, the one origin applications, nodes, +// sandboxes and self-hosted executors use. Every Core address they are given +// is derived from it; none is parsed back into an origin. +type PublicOrigin struct{ origin string } + +// NewPublicOrigin accepts a canonical public origin, never a path or // credential. It may be http or https: a reverse proxy in front of Web // terminates TLS when the installation uses it. -// OAC_PUBLIC_URL must pass it. -func ValidateCoreURL(value string) error { +func NewPublicOrigin(value string) (PublicOrigin, error) { u, err := url.Parse(value) if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" || u.Opaque != "" || u.String() != value || u.Host != strings.ToLower(u.Host) { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } if strings.ContainsAny(u.Host, "\\% \t\r\n") || strings.HasSuffix(u.Host, ":") { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } if port := u.Port(); port != "" { n, err := strconv.Atoi(port) if err != nil || n < 1 || n > 65535 || strconv.Itoa(n) != port { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } } if net.ParseIP(u.Hostname()) == nil { if len(u.Hostname()) > 253 || strings.ContainsAny(u.Host, "[]") { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } for _, label := range strings.Split(u.Hostname(), ".") { if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } for _, char := range label { if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } } } } if u.Scheme != "https" && u.Scheme != "http" { - return ErrInvalidInput + return PublicOrigin{}, ErrInvalidInput } - return nil + return PublicOrigin{origin: value}, nil } +// String is the origin itself. +func (o PublicOrigin) String() string { return o.origin } + +// API is the base URL of the Agents API. +func (o PublicOrigin) API() string { return o.origin + "/v1" } + +// RuntimeAPI is the base URL sandboxes and nodes call. +func (o PublicOrigin) RuntimeAPI() string { return o.origin + "/api/v1" } + +// DaemonWebSocket is the daemon transport: ws on an http origin, wss on https. +func (o PublicOrigin) DaemonWebSocket() string { + return "ws" + strings.TrimPrefix(o.origin, "http") + "/api/v1/agent-daemon/ws" +} + +// InstallerBase is the prefix of the versioned native installer downloads. +func (o PublicOrigin) InstallerBase() string { return o.origin + "/api/v1/agent-daemon/install/" } + // AddressBindings counts what is bound to an installation address: nodes // connect to the address they enrolled with, hosted sandboxes were started with // the address current at the time, and self-hosted executors were installed diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go index 66fd2af3f..ea2926e65 100644 --- a/services/core/internal/deployment/rules_test.go +++ b/services/core/internal/deployment/rules_test.go @@ -13,12 +13,18 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -func TestValidateCoreURL(t *testing.T) { +func TestPublicOrigin(t *testing.T) { for _, value := range []string{"https://core.example", "https://core.example:8443", "http://localhost:8091", "http://127.0.0.2:8091", "http://[::1]:8091", "https://[2001:db8::1]", "http://core.example", "http://core:8091", "http://10.0.0.5:8080"} { - if err := ValidateCoreURL(value); err != nil { + if origin, err := NewPublicOrigin(value); err != nil || origin.String() != value { t.Errorf("valid Core URL %q rejected: %v", value, err) } } + for value, socket := range map[string]string{"https://core.example": "wss://core.example/api/v1/agent-daemon/ws", "http://[::1]:8091": "ws://[::1]:8091/api/v1/agent-daemon/ws"} { + origin, err := NewPublicOrigin(value) + if err != nil || origin.DaemonWebSocket() != socket || origin.API() != value+"/v1" || origin.RuntimeAPI() != value+"/api/v1" || origin.InstallerBase() != value+"/api/v1/agent-daemon/install/" { + t.Errorf("addresses derived from %q: %+v %v", value, origin, err) + } + } for _, value := range []string{ "", "ftp://core.example", "ws://core.example", "https://core.example/", "https://user:secret@core.example", "https://core.example/path", "https://core.example?", "https://core.example?q=x", "https://core.example#x", "https://core.example#", @@ -26,7 +32,7 @@ func TestValidateCoreURL(t *testing.T) { "https://core.example\\evil", "https://[not-an-ip]", "https://-core.example", "https://core..example", "https://core_example", "https://core.example.", "https://bücher.example", "https://core.example:0443", } { - if err := ValidateCoreURL(value); !errors.Is(err, ErrInvalidInput) { + if _, err := NewPublicOrigin(value); !errors.Is(err, ErrInvalidInput) { t.Errorf("invalid Core URL %q accepted: %v", value, err) } } diff --git a/services/core/internal/environmentconfig/setup_test.go b/services/core/internal/environmentconfig/setup_test.go index 6b18c71a4..f6040532e 100644 --- a/services/core/internal/environmentconfig/setup_test.go +++ b/services/core/internal/environmentconfig/setup_test.go @@ -10,7 +10,7 @@ import ( ) func TestSetupReservesOpenAgentCoreNames(t *testing.T) { - for _, name := range []string{"OAC_ADDR", "OAC_RUNTIME_HOME", "OAC_WEB_ORIGIN", "OAC_LOG_LEVEL", "OAC_DEV_HOME", "OAC_TEST_DATABASE_URL"} { + for _, name := range []string{"OAC_ADDR", "OAC_RUNTIME_HOME", "OAC_PUBLIC_URL", "OAC_LOG_LEVEL", "OAC_DEV_HOME", "OAC_TEST_DATABASE_URL"} { if err := (Setup{Env: map[string]string{name: "value"}}).Validate(); !errors.Is(err, ErrInvalid) { t.Fatalf("reserved name %s accepted: %v", name, err) } diff --git a/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go index bb6766c52..6455c6234 100644 --- a/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go +++ b/services/core/internal/nativeinstaller/bootstrap_interrupt_unix_test.go @@ -101,7 +101,7 @@ func TestBootstrapCommandDiscardsTimedOutResponse(t *testing.T) { t.Fatal(err) } catalog := Catalog{Version: "test"} - command := exec.Command("bash", "-c", catalog.Commands(server.URL, "fixture-grant")["posix"]) + command := exec.Command("bash", "-c", catalog.Commands(server.URL+"/api/v1/agent-daemon/install/", "fixture-grant")["posix"]) command.Env = append(os.Environ(), "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") output, err := command.CombinedOutput() if err != nil || !bytes.Contains(output, []byte("entry-success")) || bytes.Contains(output, []byte("incomplete response")) || requests.Load() != 2 { diff --git a/services/core/internal/nativeinstaller/catalog.go b/services/core/internal/nativeinstaller/catalog.go index 2a40dc0a5..8c0049d88 100644 --- a/services/core/internal/nativeinstaller/catalog.go +++ b/services/core/internal/nativeinstaller/catalog.go @@ -42,8 +42,12 @@ var platformName = regexp.MustCompile(`^(linux|darwin|windows)-(amd64|arm64)$`) // Load checks the matched catalog without downloading execution payloads. Local // offline archives are verified once; the directory stays immutable while serving. +// A directory without catalog.json holds no installer and returns nil. func Load(directory, version string) (*Catalog, error) { raw, err := os.ReadFile(filepath.Join(directory, "catalog.json")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } if err != nil { return nil, err } @@ -123,8 +127,9 @@ func (c *Catalog) ServeHTTP(w http.ResponseWriter, r *http.Request) { func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } func psQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "''") + "'" } -func (c *Catalog) Commands(origin, authorization string) map[string]string { - base := origin + "/api/v1/agent-daemon/install/" + c.Version +// Commands installs this catalog's version from the installer base URL. +func (c *Catalog) Commands(installerBase, authorization string) map[string]string { + base := installerBase + c.Version // Hold the small bootstrap in memory so an interrupted fetch leaves no file. // Only execute a complete successful response; preserve interactive stdin. posix := `set -e; script=; for attempt in 1 2 3; do if script=$(curl -fsS --connect-timeout 15 --max-time 60 --max-filesize 1048576 ` + shellQuote(base+"/bootstrap.sh") + `); then break; fi; [ "$attempt" -lt 3 ] || exit 1; sleep "$attempt"; done; bash -c "$script" -- "$@"` diff --git a/services/core/internal/nativeinstaller/catalog_test.go b/services/core/internal/nativeinstaller/catalog_test.go index 5f5f9cf45..ce6fd24f3 100644 --- a/services/core/internal/nativeinstaller/catalog_test.go +++ b/services/core/internal/nativeinstaller/catalog_test.go @@ -108,3 +108,9 @@ func TestOnlineCatalogRedirectsOnlyDeclaredMatchedArchives(t *testing.T) { t.Fatal("corruption must not fall back to online download") } } + +func TestMissingCatalogServesNoInstallers(t *testing.T) { + if catalog, err := Load(t.TempDir(), "build"); catalog != nil || err != nil { + t.Fatal(catalog, err) + } +} diff --git a/services/core/internal/processconfig/config.go b/services/core/internal/processconfig/config.go index c7f1805b7..eb5eb0117 100644 --- a/services/core/internal/processconfig/config.go +++ b/services/core/internal/processconfig/config.go @@ -1,10 +1,17 @@ -// Package processconfig is the process settings Core loads from its environment. -// Startup and `oac-core check-config` both call Check. Settings reports the -// effective values for GET /core/v1/installation. Errors name the variable and -// never include its value. +// Package processconfig is the process settings Core loads from its +// environment. Load reads every variable and every file it names exactly once, +// applies each default and validates the result; startup and `oac-core +// check-config` both call it. Errors name the variable and never include its +// value or the content of a file. package processconfig import ( + "bytes" + "cmp" + "encoding/base64" + "encoding/json" + "io" + "net/url" "os" "slices" "strconv" @@ -12,154 +19,264 @@ import ( "time" "github.com/google/uuid" + "golang.org/x/net/http/httpguts" + "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/databaseurl" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/oauthrefresh" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -// Check validates every process setting Core loads. An unset or empty -// variable keeps its default, so Compose can pass every setting through. -func Check() error { - if _, err := PublicURL(); err != nil { - return err +const ( + defaultAddr = "127.0.0.1:8091" + defaultHarness = "codex" + defaultWriteAuditRetention = 90 * 24 * time.Hour + // providerStateRoot is where Compose mounts the data volume's state/. + providerStateRoot = "/state" +) + +// Config is Core's process configuration. +type Config struct { + // Addr is OAC_ADDR, the listener address. + Addr string + // PublicOrigin is OAC_PUBLIC_URL. Nil disables the Runtime gateway and + // the execution Worker. + PublicOrigin *deployment.PublicOrigin + // DatabaseURL is OAC_DATABASE_URL with the password from + // OAC_DATABASE_PASSWORD_FILE. + DatabaseURL string + // InstallationID comes from OAC_INSTALLATION_ID_FILE. Empty leaves the + // sandbox deployment and node routes off. + InstallationID string + // CredentialKey seals stored credentials. Nil when + // OAC_CREDENTIAL_KEY_FILE is unset. + CredentialKey *credentialcrypto.Cipher + // CoreKeys authenticates the Core key from OAC_CORE_KEY_DIGESTS_FILE. + CoreKeys *api.DeploymentAuthenticator + ExecutionConcurrency int + DefaultHarness string + // Harnesses is the sorted set of enabled Harnesses, the default included. + Harnesses []string + WriteAuditRetention time.Duration + OAuthTrustedOrigins []string + RuntimeHistory RuntimeHistory + // ProviderPaths locate adapter helpers under OAC_PROVIDER_ROOT and their + // private state under the Compose state mount. + ProviderPaths sandbox.ProcessPaths + // NativeInstallers is the native installer catalog directory under + // OAC_PROVIDER_ROOT; empty when the root is unset. + NativeInstallers string + Log log.Config +} + +// RuntimeHistory is the file named by OAC_HISTORY_SETTINGS_FILE, with its +// defaults applied. Without the file, history stays in Core's database and +// nothing is exported. +type RuntimeHistory struct { + // File is the path Core loaded, or empty. + File string + // Endpoint is an optional OTLP/HTTP metrics URL; Insecure allows http. + Endpoint string + Insecure bool + Headers map[string]string + QueueCapacity int + Timeout time.Duration + SampleInterval time.Duration +} + +// runtimeHistoryFile is the file's JSON schema. +type runtimeHistoryFile struct { + Transport string `json:"transport,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Insecure bool `json:"insecure,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + QueueCapacity int `json:"queue_capacity,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + SampleIntervalSeconds int `json:"sample_interval_seconds,omitempty"` +} + +// Load reads and validates the process environment. An unset or empty +// variable selects its default. +func Load() (Config, error) { + var c Config + var err error + if c.Log, err = log.LoadConfig(); err != nil { + return Config{}, err } - if _, err := ExecutionConcurrency(); err != nil { - return err + c.Addr = cmp.Or(os.Getenv("OAC_ADDR"), defaultAddr) + if value := os.Getenv("OAC_PUBLIC_URL"); value != "" { + origin, err := deployment.NewPublicOrigin(value) + if err != nil { + return Config{}, configError("OAC_PUBLIC_URL must be a canonical http or https origin without path, credentials, query or fragment, such as https://core.example") + } + c.PublicOrigin = &origin } - if _, err := InstallationID(); err != nil { - return err + if c.DatabaseURL, err = databaseurl.FromEnvironment(); err != nil { + return Config{}, err } - engineName, err := DefaultHarness() - if err != nil { - return err - } - if _, err := Harnesses(engineName); err != nil { - return err - } - if _, err := writeAuditRetention(); err != nil { - return err - } - if err := oauthOrigins(); err != nil { - return err - } - return logSettings() -} - -// Settings is the effective process configuration. Sensitive file settings -// report only whether they are configured. -func Settings() ([]api.InstallationSetting, error) { - if err := Check(); err != nil { - return nil, err - } - public, _ := PublicURL() - concurrency, _ := ExecutionConcurrency() - engineName, _ := DefaultHarness() - enabled, _ := Harnesses(engineName) - retention := "2160h" - if value := os.Getenv("OAC_WRITE_AUDIT_RETENTION"); value != "" { - retention = value - } - level, format, addSource := logValues() - history := os.Getenv("OAC_HISTORY_SETTINGS_FILE") != "" - origins := []string{} - if raw := os.Getenv("OAC_OAUTH_TRUSTED_ORIGINS"); raw != "" { - for _, origin := range strings.Split(raw, ",") { - if origin = strings.TrimSpace(origin); origin != "" { - origins = append(origins, origin) - } - } + if c.DatabaseURL == "" { + return Config{}, configError("OAC_DATABASE_URL is required") + } + if c.InstallationID, err = installationID(); err != nil { + return Config{}, err + } + if c.InstallationID != "" && c.PublicOrigin == nil { + return Config{}, configError("OAC_INSTALLATION_ID_FILE requires OAC_PUBLIC_URL, the origin nodes and sandboxes use to reach Core") + } + if c.CredentialKey, err = credentialKey(); err != nil { + return Config{}, err + } + if c.CoreKeys, err = coreKeys(); err != nil { + return Config{}, err + } + if c.ExecutionConcurrency, err = executionConcurrency(); err != nil { + return Config{}, err + } + c.DefaultHarness = cmp.Or(os.Getenv("OAC_DEFAULT_HARNESS"), defaultHarness) + if _, known := (engine.Catalog{}).Lookup(c.DefaultHarness); !known { + return Config{}, configError("OAC_DEFAULT_HARNESS is not a known harness") + } + if c.Harnesses, err = harnesses(c.DefaultHarness); err != nil { + return Config{}, err + } + if c.WriteAuditRetention, err = writeAuditRetention(); err != nil { + return Config{}, err + } + if c.OAuthTrustedOrigins, err = oauthTrustedOrigins(); err != nil { + return Config{}, err } - var publicValue any - if public != "" { - publicValue = public + if c.RuntimeHistory, err = runtimeHistory(); err != nil { + return Config{}, err + } + c.ProviderPaths = sandbox.ProcessPaths{ArtifactRoot: os.Getenv("OAC_PROVIDER_ROOT"), StateRoot: providerStateRoot} + if c.ProviderPaths.ArtifactRoot != "" { + c.NativeInstallers = c.ProviderPaths.ArtifactRoot + "/native-installers" + } + return c, nil +} + +// Settings projects the configuration that GET /core/v1/installation +// reports. Sensitive file settings report only whether they are configured. +func (c Config) Settings() []api.InstallationSetting { + var public any + if c.PublicOrigin != nil { + public = c.PublicOrigin.String() + } + format := c.Log.Format + if format == "" { + format = "auto" + } + origins := c.OAuthTrustedOrigins + if origins == nil { + origins = []string{} } return []api.InstallationSetting{ - setting("public_url", publicValue, nil, true, []string{"core", "web"}), - setting("log.level", level, "info", true, []string{"core", "web"}), - setting("log.format", format, "auto", true, []string{"core", "web"}), - setting("log.add_source", addSource, false, true, []string{"core", "web"}), - setting("core.execution_concurrency", concurrency, execution.DefaultExecutionConcurrency, true, []string{"core"}), - setting("core.harnesses", enabled, (engine.Catalog{}).Kinds(), true, []string{"core"}), - setting("core.default_harness", engineName, "codex", true, []string{"core"}), - setting("core.write_audit_retention", retention, "2160h", true, []string{"core"}), - setting("core.oauth_trusted_origins", origins, []string{}, true, []string{"core"}), - sensitive("core.runtime_history", history, []string{"core"}), - }, nil + setting("public_url", public, nil, []string{"core", "web"}), + setting("log.level", strings.ToLower(c.Log.Level.String()), "info", []string{"core", "web"}), + setting("log.format", format, "auto", []string{"core", "web"}), + setting("log.add_source", c.Log.AddSource, false, []string{"core", "web"}), + setting("core.execution_concurrency", c.ExecutionConcurrency, execution.DefaultExecutionConcurrency, []string{"core"}), + setting("core.harnesses", c.Harnesses, (engine.Catalog{}).Kinds(), []string{"core"}), + setting("core.default_harness", c.DefaultHarness, defaultHarness, []string{"core"}), + setting("core.write_audit_retention", duration(c.WriteAuditRetention), duration(defaultWriteAuditRetention), []string{"core"}), + setting("core.oauth_trusted_origins", origins, []string{}, []string{"core"}), + sensitive("core.runtime_history", c.RuntimeHistory.File != "", []string{"core"}), + } } -func setting(key string, value, fallback any, changeable bool, restarts []string) api.InstallationSetting { - return api.InstallationSetting{Key: key, Value: value, Default: fallback, Changeable: changeable, Sensitive: false, Restarts: restarts} +// duration formats d as OAC_WRITE_AUDIT_RETENTION spells it: 2160h, not +// 2160h0m0s. +func duration(d time.Duration) string { + s := d.String() + if strings.HasSuffix(s, "m0s") { + s = s[:len(s)-2] + } + if strings.HasSuffix(s, "h0m") { + s = s[:len(s)-2] + } + return s +} + +func setting(key string, value, fallback any, restarts []string) api.InstallationSetting { + return api.InstallationSetting{Key: key, Value: value, Default: fallback, Changeable: true, Sensitive: false, Restarts: restarts} } func sensitive(key string, configured bool, restarts []string) api.InstallationSetting { return api.InstallationSetting{Key: key, Configured: &configured, Changeable: true, Sensitive: true, Restarts: restarts} } -// PublicURL reads OAC_PUBLIC_URL, the one origin applications, nodes, -// sandboxes and self-hosted executors use. An empty result disables daemon -// transport, as for a Core without execution. -func PublicURL() (string, error) { - value := os.Getenv("OAC_PUBLIC_URL") - if value == "" { +func installationID() (string, error) { + path := os.Getenv("OAC_INSTALLATION_ID_FILE") + if path == "" { return "", nil } - if deployment.ValidateCoreURL(value) != nil { - return "", configErr("OAC_PUBLIC_URL must be a canonical http or https origin without path, credentials, query or fragment, such as https://core.example") + raw, err := os.ReadFile(path) + if err != nil { + return "", configError("OAC_INSTALLATION_ID_FILE must name a readable file") + } + value := strings.TrimSpace(string(raw)) + if id, err := uuid.Parse(value); err != nil || id == uuid.Nil || id.String() != value { + return "", configError("OAC_INSTALLATION_ID_FILE must contain a canonical UUID") } return value, nil } -// InstallationID reads the file named by OAC_INSTALLATION_ID_FILE. The ID -// enables the sandbox deployment and node routes; unset leaves them off. -func InstallationID() (string, error) { - path := os.Getenv("OAC_INSTALLATION_ID_FILE") +func credentialKey() (*credentialcrypto.Cipher, error) { + path := os.Getenv("OAC_CREDENTIAL_KEY_FILE") if path == "" { - return "", nil + return nil, nil + } + content, err := os.ReadFile(path) + if err != nil { + return nil, configError("cannot read OAC_CREDENTIAL_KEY_FILE") + } + key, err := base64.StdEncoding.Strict().DecodeString(strings.TrimSpace(string(content))) + if err != nil || len(key) != 32 { + return nil, configError("OAC_CREDENTIAL_KEY_FILE must contain a base64-encoded random 32-byte key") + } + return credentialcrypto.New(key) +} + +func coreKeys() (*api.DeploymentAuthenticator, error) { + path := os.Getenv("OAC_CORE_KEY_DIGESTS_FILE") + if path == "" { + return nil, configError("OAC_CORE_KEY_DIGESTS_FILE is required; Core needs the Core key digest") } raw, err := os.ReadFile(path) if err != nil { - return "", configErr("OAC_INSTALLATION_ID_FILE must name a readable file") + return nil, configError("cannot read OAC_CORE_KEY_DIGESTS_FILE") } - value := strings.TrimSpace(string(raw)) - if id, err := uuid.Parse(value); err != nil || id == uuid.Nil || id.String() != value { - return "", configErr("OAC_INSTALLATION_ID_FILE must contain a canonical UUID") + var digests []string + if json.Unmarshal(raw, &digests) != nil { + return nil, configError("OAC_CORE_KEY_DIGESTS_FILE must contain a JSON array of Core key SHA-256 digests") } - return value, nil + keys, err := api.NewDeploymentAuthenticator(digests) + if err != nil { + return nil, configError("OAC_CORE_KEY_DIGESTS_FILE must contain a JSON array of Core key SHA-256 digests") + } + return keys, nil } -// ExecutionConcurrency reads OAC_EXECUTION_CONCURRENCY. Unset or empty keeps -// the default. -func ExecutionConcurrency() (int, error) { +func executionConcurrency() (int, error) { value := os.Getenv("OAC_EXECUTION_CONCURRENCY") if value == "" { return execution.DefaultExecutionConcurrency, nil } limit, err := strconv.Atoi(value) if err != nil || limit < 1 || limit > 1024 { - return 0, configErr("OAC_EXECUTION_CONCURRENCY must be an integer between 1 and 1024") + return 0, configError("OAC_EXECUTION_CONCURRENCY must be an integer between 1 and 1024") } return limit, nil } -// DefaultHarness reads OAC_DEFAULT_HARNESS, the Harness used when a request -// names none. -func DefaultHarness() (string, error) { - value := os.Getenv("OAC_DEFAULT_HARNESS") - if value == "" { - return "codex", nil - } - if _, known := (engine.Catalog{}).Lookup(value); !known { - return "", configErr("OAC_DEFAULT_HARNESS is not a known harness") - } - return value, nil -} - -// Harnesses reads OAC_HARNESSES. Unset enables every Harness this build +// harnesses reads OAC_HARNESSES. Unset enables every Harness this build // supports; a list supplements the default Harness. -func Harnesses(defaultEngine string) ([]string, error) { +func harnesses(defaultEngine string) ([]string, error) { kinds := []string{defaultEngine} if value := os.Getenv("OAC_HARNESSES"); value != "" { kinds = append(kinds, strings.Split(value, ",")...) @@ -169,7 +286,7 @@ func Harnesses(defaultEngine string) ([]string, error) { for i, kind := range kinds { kind = strings.TrimSpace(kind) if _, known := (engine.Catalog{}).Lookup(kind); !known { - return nil, configErr("OAC_HARNESSES contains an unknown harness") + return nil, configError("OAC_HARNESSES contains an unknown harness") } kinds[i] = kind } @@ -180,55 +297,97 @@ func Harnesses(defaultEngine string) ([]string, error) { func writeAuditRetention() (time.Duration, error) { value := os.Getenv("OAC_WRITE_AUDIT_RETENTION") if value == "" { - return 90 * 24 * time.Hour, nil + return defaultWriteAuditRetention, nil } - duration, parseErr := time.ParseDuration(value) - if parseErr != nil || duration < time.Hour { - return 0, configErr("OAC_WRITE_AUDIT_RETENTION must be a duration of at least 1h") + duration, err := time.ParseDuration(value) + if err != nil || duration < time.Hour { + return 0, configError("OAC_WRITE_AUDIT_RETENTION must be a duration of at least 1h") } return duration, nil } -func oauthOrigins() error { - var origins []string - if raw := os.Getenv("OAC_OAUTH_TRUSTED_ORIGINS"); raw != "" { - for _, origin := range strings.Split(raw, ",") { - origins = append(origins, strings.TrimSpace(origin)) - } +func oauthTrustedOrigins() ([]string, error) { + raw := os.Getenv("OAC_OAUTH_TRUSTED_ORIGINS") + if raw == "" { + return nil, nil + } + origins := strings.Split(raw, ",") + for i, origin := range origins { + origins[i] = strings.TrimSpace(origin) } if _, err := oauthrefresh.NewClient(origins); err != nil { - return configErr("OAC_OAUTH_TRUSTED_ORIGINS is invalid") + return nil, configError("OAC_OAUTH_TRUSTED_ORIGINS is invalid") } - return nil + return origins, nil } -func logSettings() error { - if value, ok := os.LookupEnv("OAC_LOG_LEVEL"); ok && value != "" && !slices.Contains([]string{"debug", "info", "warn", "warning", "error", "err"}, strings.ToLower(strings.TrimSpace(value))) { - return configErr("OAC_LOG_LEVEL must be debug, info, warn or error") - } - if value, ok := os.LookupEnv("OAC_LOG_FORMAT"); ok && value != "" && !slices.Contains([]string{"auto", "json", "text"}, strings.ToLower(strings.TrimSpace(value))) { - return configErr("OAC_LOG_FORMAT must be auto, json or text") +func runtimeHistory() (RuntimeHistory, error) { + var file runtimeHistoryFile + path := os.Getenv("OAC_HISTORY_SETTINGS_FILE") + if path != "" { + raw, err := os.ReadFile(path) + if err != nil { + return RuntimeHistory{}, configError("OAC_HISTORY_SETTINGS_FILE: the file cannot be read") + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if decoder.Decode(&file) != nil || decoder.Decode(new(any)) != io.EOF { + return RuntimeHistory{}, configError("OAC_HISTORY_SETTINGS_FILE: the file must hold one JSON object with known fields") + } } - if value, ok := os.LookupEnv("OAC_LOG_ADD_SOURCE"); ok && value != "" && value != "0" && value != "1" { - return configErr("OAC_LOG_ADD_SOURCE must be 0 or 1") + if err := validateRuntimeHistory(file); err != nil { + return RuntimeHistory{}, err } - return nil + return RuntimeHistory{File: path, Endpoint: file.Endpoint, Insecure: file.Insecure, Headers: file.Headers, + QueueCapacity: cmp.Or(file.QueueCapacity, 256), + Timeout: time.Duration(cmp.Or(file.TimeoutSeconds, 2)) * time.Second, + SampleInterval: time.Duration(cmp.Or(file.SampleIntervalSeconds, 30)) * time.Second}, nil } -func logValues() (string, string, bool) { - level := "info" - if value := strings.ToLower(strings.TrimSpace(os.Getenv("OAC_LOG_LEVEL"))); value != "" { - level = value +func validateRuntimeHistory(file runtimeHistoryFile) error { + if file.QueueCapacity < 0 || file.QueueCapacity > 4096 { + return configError("OAC_HISTORY_SETTINGS_FILE: queue_capacity must be from 0 to 4096") + } + if file.TimeoutSeconds < 0 || file.TimeoutSeconds > 30 { + return configError("OAC_HISTORY_SETTINGS_FILE: timeout_seconds must be from 0 to 30") } - format := "auto" - if value := strings.ToLower(strings.TrimSpace(os.Getenv("OAC_LOG_FORMAT"))); value != "" { - format = value + if file.SampleIntervalSeconds != 0 && (file.SampleIntervalSeconds < 5 || file.SampleIntervalSeconds > 300) { + return configError("OAC_HISTORY_SETTINGS_FILE: sample_interval_seconds must be from 5 to 300") + } + if file.Endpoint == "" { + if file.Transport != "" || file.Insecure || len(file.Headers) != 0 { + return configError("OAC_HISTORY_SETTINGS_FILE: transport, insecure and headers require an endpoint") + } + return nil } - return level, format, os.Getenv("OAC_LOG_ADD_SOURCE") == "1" + if file.Transport != "otlp_http" { + return configError("OAC_HISTORY_SETTINGS_FILE: transport must be otlp_http") + } + endpoint, err := url.Parse(file.Endpoint) + if err != nil || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" || endpoint.RawPath != "" || endpoint.Path == "" || endpoint.String() != file.Endpoint { + return configError("OAC_HISTORY_SETTINGS_FILE: endpoint must be a canonical absolute OTLP metrics URL") + } + switch endpoint.Scheme { + case "https": + if file.Insecure { + return configError("OAC_HISTORY_SETTINGS_FILE: insecure requires an http endpoint") + } + case "http": + if !file.Insecure { + return configError("OAC_HISTORY_SETTINGS_FILE: an http endpoint requires insecure=true") + } + default: + return configError("OAC_HISTORY_SETTINGS_FILE: endpoint scheme must be https or explicit insecure http") + } + for key, value := range file.Headers { + lower := strings.ToLower(key) + if !httpguts.ValidHeaderFieldName(key) || !httpguts.ValidHeaderFieldValue(value) || lower == "host" || lower == "content-length" || lower == "content-type" || lower == "content-encoding" { + return configError("OAC_HISTORY_SETTINGS_FILE: headers contain an invalid or reserved entry") + } + } + return nil } type configError string func (e configError) Error() string { return string(e) } - -func configErr(message string) error { return configError(message) } diff --git a/services/core/internal/processconfig/config_test.go b/services/core/internal/processconfig/config_test.go index 9f8dfa533..d66edb2bc 100644 --- a/services/core/internal/processconfig/config_test.go +++ b/services/core/internal/processconfig/config_test.go @@ -1,69 +1,235 @@ package processconfig import ( + "bytes" + "encoding/base64" + "os" + "path/filepath" "strings" "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" ) -func TestCheckRejectsInvalidValuesWithoutEchoingThem(t *testing.T) { - secret := "https://user:synthetic-secret@core.example" - t.Setenv("OAC_PUBLIC_URL", secret) - err := Check() - if err == nil || strings.Contains(err.Error(), "synthetic-secret") || !strings.Contains(err.Error(), "OAC_PUBLIC_URL") { +// required sets the settings Load requires and returns a file writer. +func required(t *testing.T) func(name, content string) string { + t.Helper() + dir := t.TempDir() + write := func(name, content string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path + } + t.Setenv("OAC_DATABASE_URL", "postgres://core@database/core") + t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", write("digests.json", `["`+strings.Repeat("ab", 32)+`"]`)) + return write +} + +// rejects asserts that Load fails, names variable and does not echo secret. +func rejects(t *testing.T, variable, secret string) { + t.Helper() + _, err := Load() + if err == nil || !strings.Contains(err.Error(), variable) || (secret != "" && strings.Contains(err.Error(), secret)) { + t.Fatalf("%s: %v", variable, err) + } +} + +func TestLoadAppliesDefaults(t *testing.T) { + required(t) + c, err := Load() + if err != nil { + t.Fatal(err) + } + if c.Addr != "127.0.0.1:8091" || c.PublicOrigin != nil || c.InstallationID != "" || c.CredentialKey != nil || c.CoreKeys == nil || + c.ExecutionConcurrency != 4 || c.DefaultHarness != "codex" || strings.Join(c.Harnesses, ",") != "claude_sdk,codex,mcode" || + c.WriteAuditRetention != 90*24*time.Hour || c.OAuthTrustedOrigins != nil || c.NativeInstallers != "" || c.ProviderPaths.StateRoot != "/state" { + t.Fatalf("%+v", c) + } + if h := c.RuntimeHistory; h.File != "" || h.Endpoint != "" || h.QueueCapacity != 256 || h.Timeout != 2*time.Second || h.SampleInterval != 30*time.Second { + t.Fatalf("%+v", h) + } + t.Setenv("OAC_PROVIDER_ROOT", "/opt/oac") + if c, err = Load(); err != nil || c.ProviderPaths.ArtifactRoot != "/opt/oac" || c.NativeInstallers != "/opt/oac/native-installers" { + t.Fatal(c, err) + } +} + +func TestLoadRejectsInvalidValuesWithoutEchoingThem(t *testing.T) { + write := required(t) + t.Setenv("OAC_DATABASE_URL", "") + rejects(t, "OAC_DATABASE_URL", "") + required(t) + t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", "") + rejects(t, "OAC_CORE_KEY_DIGESTS_FILE", "") + t.Setenv("OAC_CORE_KEY_DIGESTS_FILE", write("bad-digests.json", `["synthetic-secret"]`)) + rejects(t, "OAC_CORE_KEY_DIGESTS_FILE", "synthetic-secret") + required(t) + for variable, value := range map[string]string{ + "OAC_PUBLIC_URL": "https://user:synthetic-secret@core.example", + "OAC_EXECUTION_CONCURRENCY": "synthetic-secret", + "OAC_DEFAULT_HARNESS": "synthetic-secret", + "OAC_HARNESSES": "codex,synthetic-secret", + "OAC_WRITE_AUDIT_RETENTION": "synthetic-secret", + "OAC_OAUTH_TRUSTED_ORIGINS": "https://synthetic-secret.example/token", + "OAC_LOG_LEVEL": "verbose", + "OAC_INSTALLATION_ID_FILE": write("installation.id", "synthetic-secret"), + "OAC_CREDENTIAL_KEY_FILE": write("credential.key", "synthetic-secret"), + "OAC_HISTORY_SETTINGS_FILE": write("history.json", `{"secret":"synthetic-secret"}`), + } { + t.Run(variable, func(t *testing.T) { + t.Setenv(variable, value) + rejects(t, variable, "synthetic-secret") + }) + } + t.Setenv("OAC_INSTALLATION_ID_FILE", write("valid.id", "8c5f4f5e-2c55-4c43-9a49-7f2f3f2d1d10\n")) + rejects(t, "OAC_PUBLIC_URL", "") + t.Setenv("OAC_PUBLIC_URL", "https://core.example") + if c, err := Load(); err != nil || c.InstallationID != "8c5f4f5e-2c55-4c43-9a49-7f2f3f2d1d10" { + t.Fatal(c.InstallationID, err) + } +} + +func TestPublicURLMustBeACanonicalOrigin(t *testing.T) { + required(t) + for _, value := range []string{"https://core.example", "https://core.example:8443", "http://127.0.0.1:8091", "http://core.example"} { + t.Setenv("OAC_PUBLIC_URL", value) + if c, err := Load(); err != nil || c.PublicOrigin.String() != value { + t.Fatal(value, err) + } + } + for _, value := range []string{"https://core.example/", "https://Core.example", "wss://core.example", "https://core.example/v1"} { + t.Setenv("OAC_PUBLIC_URL", value) + rejects(t, "OAC_PUBLIC_URL", "") + } +} + +func TestExecutionConcurrencyAndAuditRetention(t *testing.T) { + required(t) + for _, value := range []string{"1", "1024"} { + t.Setenv("OAC_EXECUTION_CONCURRENCY", value) + if _, err := Load(); err != nil { + t.Fatal(value, err) + } + } + for _, value := range []string{"0", "-1", "1025", "1.5"} { + t.Setenv("OAC_EXECUTION_CONCURRENCY", value) + rejects(t, "OAC_EXECUTION_CONCURRENCY", "") + } + t.Setenv("OAC_EXECUTION_CONCURRENCY", "") + t.Setenv("OAC_WRITE_AUDIT_RETENTION", "24h") + if c, err := Load(); err != nil || c.WriteAuditRetention != 24*time.Hour { + t.Fatal(c.WriteAuditRetention, err) + } + for _, value := range []string{"0", "30m", "-1h", "90d"} { + t.Setenv("OAC_WRITE_AUDIT_RETENTION", value) + rejects(t, "OAC_WRITE_AUDIT_RETENTION", "") + } +} + +func TestHarnessesDefaultToEveryQualifiedHarness(t *testing.T) { + required(t) + t.Setenv("OAC_HARNESSES", "mcode") + if c, err := Load(); err != nil || strings.Join(c.Harnesses, ",") != "codex,mcode" { + t.Fatal(c.Harnesses, err) + } +} + +func TestOAuthTrustedOrigins(t *testing.T) { + required(t) + t.Setenv("OAC_OAUTH_TRUSTED_ORIGINS", "https://issuer.example, https://10.0.0.1:9443") + if c, err := Load(); err != nil || strings.Join(c.OAuthTrustedOrigins, ",") != "https://issuer.example,https://10.0.0.1:9443" { + t.Fatal(c.OAuthTrustedOrigins, err) + } + for _, value := range []string{"http://issuer.example", "https://issuer.example/token", "https://issuer.example,", "https://user:secret@issuer.example"} { + t.Setenv("OAC_OAUTH_TRUSTED_ORIGINS", value) + rejects(t, "OAC_OAUTH_TRUSTED_ORIGINS", "") + } +} + +func TestCredentialKey(t *testing.T) { + write := required(t) + t.Setenv("OAC_CREDENTIAL_KEY_FILE", filepath.Join(t.TempDir(), "missing.key")) + rejects(t, "OAC_CREDENTIAL_KEY_FILE", "") + for _, content := range []string{"", base64.StdEncoding.EncodeToString(make([]byte, 31))} { + t.Setenv("OAC_CREDENTIAL_KEY_FILE", write("short.key", content)) + rejects(t, "OAC_CREDENTIAL_KEY_FILE", "") + } + t.Setenv("OAC_CREDENTIAL_KEY_FILE", write("credential.key", base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{0x91}, 32))+"\n")) + first, err := Load() + if err != nil { t.Fatal(err) } - t.Setenv("OAC_PUBLIC_URL", "") - t.Setenv("OAC_EXECUTION_CONCURRENCY", "0") - if err := Check(); err == nil || !strings.Contains(err.Error(), "OAC_EXECUTION_CONCURRENCY") || strings.Contains(err.Error(), "synthetic") { + binding := credentialcrypto.Binding{TenantID: "tenant", VaultID: "vault", CredentialID: "credential", AuthType: "static_bearer", Destination: "https://example.invalid/mcp"} + sealed, err := first.CredentialKey.Seal([]byte("opaque storage test"), binding) + if err != nil { t.Fatal(err) } - t.Setenv("OAC_EXECUTION_CONCURRENCY", "4") - t.Setenv("OAC_LOG_LEVEL", "verbose") - if err := Check(); err == nil || !strings.Contains(err.Error(), "OAC_LOG_LEVEL") { + reopened, err := Load() + if err != nil { t.Fatal(err) } + if got, err := reopened.CredentialKey.Open(sealed, binding); err != nil || string(got) != "opaque storage test" { + t.Fatal("persisted key did not recover ciphertext", err) + } +} + +func TestRuntimeHistoryFile(t *testing.T) { + write := required(t) + t.Setenv("OAC_HISTORY_SETTINGS_FILE", write("history.json", `{"transport":"otlp_http","endpoint":"http://127.0.0.1:4318/v1/metrics","insecure":true,"headers":{"X-Scope-OrgID":"operator-history"},"sample_interval_seconds":60}`)) + c, err := Load() + if err != nil || c.RuntimeHistory.Endpoint != "http://127.0.0.1:4318/v1/metrics" || c.RuntimeHistory.SampleInterval != time.Minute || c.RuntimeHistory.QueueCapacity != 256 { + t.Fatal(c.RuntimeHistory, err) + } + for name, config := range map[string]string{ + "unknown field": `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","secret":"must-not-leak"}`, + "implicit insecure": `{"transport":"otlp_http","endpoint":"http://collector.example.test/v1/metrics"}`, + "userinfo": `{"transport":"otlp_http","endpoint":"https://user:must-not-leak@collector.example.test/v1/metrics"}`, + "header newline": "{\"transport\":\"otlp_http\",\"endpoint\":\"https://collector.example.test/v1/metrics\",\"headers\":{\"Authorization\":\"Bearer must-not-leak\\n\"}}", + "reserved header": `{"transport":"otlp_http","endpoint":"https://collector.example.test/v1/metrics","headers":{"Host":"must-not-leak"}}`, + "oversized queue": `{"queue_capacity":4097}`, + "oversized timeout": `{"timeout_seconds":31}`, + "too frequent sampling": `{"sample_interval_seconds":4}`, + "oversized sampling interval": `{"sample_interval_seconds":301}`, + "transport without endpoint": `{"transport":"otlp_http"}`, + "trailing value": `{} {}`, + } { + t.Run(name, func(t *testing.T) { + t.Setenv("OAC_HISTORY_SETTINGS_FILE", write("invalid.json", config)) + if _, err := Load(); err == nil || !strings.HasPrefix(err.Error(), "OAC_HISTORY_SETTINGS_FILE: ") || strings.Contains(err.Error(), "must-not-leak") { + t.Fatal(err) + } + }) + } } func TestSettingsReportEffectiveValuesAndHideHistory(t *testing.T) { + write := required(t) t.Setenv("OAC_PUBLIC_URL", "https://core.example") t.Setenv("OAC_EXECUTION_CONCURRENCY", "8") - t.Setenv("OAC_HISTORY_SETTINGS_FILE", "/tmp/history.json") - settings, err := Settings() + t.Setenv("OAC_LOG_LEVEL", "warn") + t.Setenv("OAC_WRITE_AUDIT_RETENTION", "1440m") + t.Setenv("OAC_HISTORY_SETTINGS_FILE", write("history.json", `{}`)) + c, err := Load() if err != nil { t.Fatal(err) } found := map[string]any{} - for _, setting := range settings { + for _, setting := range c.Settings() { if setting.Sensitive && (setting.Value != nil || setting.Configured == nil) { t.Fatalf("sensitive setting %s leaked a value", setting.Key) } found[setting.Key] = setting.Value + if setting.Key == "core.write_audit_retention" && setting.Default != "2160h" { + t.Fatal(setting.Default) + } if setting.Key == "core.runtime_history" && (setting.Configured == nil || !*setting.Configured) { t.Fatal("history file was not reported as configured") } } - if found["public_url"] != "https://core.example" || found["core.execution_concurrency"] != 8 { + if found["public_url"] != "https://core.example" || found["core.execution_concurrency"] != 8 || found["log.level"] != "warn" || found["log.format"] != "auto" || found["core.write_audit_retention"] != "24h" { t.Fatal(found) } } - -func TestHarnessesDefaultToEveryQualifiedHarness(t *testing.T) { - t.Setenv("OAC_DEFAULT_HARNESS", "") - t.Setenv("OAC_HARNESSES", "") - engineName, err := DefaultHarness() - if err != nil || engineName != "codex" { - t.Fatal(engineName, err) - } - kinds, err := Harnesses(engineName) - if err != nil || strings.Join(kinds, ",") != "claude_sdk,codex,mcode" { - t.Fatal(kinds, err) - } - t.Setenv("OAC_HARNESSES", "mcode") - if kinds, err = Harnesses("codex"); err != nil || strings.Join(kinds, ",") != "codex,mcode" { - t.Fatal(kinds, err) - } - t.Setenv("OAC_HARNESSES", "unqualified") - if _, err = Harnesses("codex"); err == nil { - t.Fatal("unqualified harness enabled") - } -} diff --git a/services/core/internal/runtime/gateway.go b/services/core/internal/runtime/gateway.go index 56a4a06e1..51c5b1163 100644 --- a/services/core/internal/runtime/gateway.go +++ b/services/core/internal/runtime/gateway.go @@ -4,7 +4,6 @@ package runtime import ( "errors" "net/http" - "net/url" "github.com/go-chi/chi/v5" @@ -17,9 +16,8 @@ import ( // receipts through cancellations. Its credentials never grant public Session // API access. func NewGateway(credentials runtimegateway.RuntimeStore, heartbeat runtimegateway.HeartbeatTouch, cancellations runtimegateway.ArchivedCancellationStore, publicWSURL string) (http.Handler, *runtimegateway.Registry, error) { - u, err := url.Parse(publicWSURL) - if err != nil || credentials == nil || heartbeat == nil || cancellations == nil || (u.Scheme != "ws" && u.Scheme != "wss") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "/api/v1/agent-daemon/ws" { - return nil, nil, errors.New("daemon URL must be an absolute ws(s) URL ending in /api/v1/agent-daemon/ws") + if credentials == nil || heartbeat == nil || cancellations == nil { + return nil, nil, errors.New("daemon gateway dependencies are required") } registry := runtimegateway.NewRegistry() h := runtimegateway.NewHandler(runtimegateway.HandlerConfig{ diff --git a/services/core/internal/runtimeobs/sampler.go b/services/core/internal/runtimeobs/sampler.go index 2fa81163f..b55e205db 100644 --- a/services/core/internal/runtimeobs/sampler.go +++ b/services/core/internal/runtimeobs/sampler.go @@ -38,11 +38,9 @@ type OwnershipChecker interface { } type SamplerOptions struct { - Interval time.Duration PageSize int Concurrency int SourceTimeout time.Duration - Report func(SweepResult) } // SweepResult is deliberately low-cardinality. It reports collection coverage @@ -54,21 +52,17 @@ type SweepResult struct { } type Sampler struct { - lister SessionLister - observer HistoryObserver - owner OwnershipChecker - options SamplerOptions - now func() time.Time - afterSweep func(SweepResult) + lister SessionLister + observer HistoryObserver + owner OwnershipChecker + options SamplerOptions + now func() time.Time } func NewSampler(lister SessionLister, observer HistoryObserver, owner OwnershipChecker, options SamplerOptions) (*Sampler, error) { if lister == nil || observer == nil || owner == nil { return nil, errors.New("Runtime history sampler dependencies are required") } - if options.Interval <= 0 { - return nil, errors.New("Runtime history sampler interval must be positive") - } if options.PageSize == 0 { options.PageSize = defaultSamplerPageSize } @@ -87,37 +81,12 @@ func NewSampler(lister SessionLister, observer HistoryObserver, owner OwnershipC if options.SourceTimeout < time.Millisecond || options.SourceTimeout > 30*time.Second { return nil, errors.New("Runtime history sampler source timeout is out of range") } - return &Sampler{lister: lister, observer: observer, owner: owner, options: options, now: time.Now, afterSweep: options.Report}, nil -} - -// Run performs one immediate full keyset sweep and then repeats without overlap. -// A failed sweep is isolated from execution and retried at the next interval. -func (s *Sampler) Run(ctx context.Context) error { - for { - result := s.sweep(ctx) - s.report(result) - if err := ctx.Err(); err != nil { - return err - } - timer := time.NewTimer(s.options.Interval) - select { - case <-timer.C: - case <-ctx.Done(): - timer.Stop() - return ctx.Err() - } - } -} - -func (s *Sampler) report(result SweepResult) { - if s.afterSweep == nil { - return - } - defer func() { _ = recover() }() - s.afterSweep(result) + return &Sampler{lister: lister, observer: observer, owner: owner, options: options, now: time.Now}, nil } -func (s *Sampler) sweep(ctx context.Context) (result SweepResult) { +// Sweep performs one full keyset sweep. A failed sweep is isolated from +// execution; the caller repeats sweeps without overlap. +func (s *Sampler) Sweep(ctx context.Context) (result SweepResult) { result.StartedAt = s.now() defer func() { result.CompletedAt = s.now() }() sweepCtx, cancel := context.WithCancel(ctx) diff --git a/services/core/internal/runtimeobs/sampler_test.go b/services/core/internal/runtimeobs/sampler_test.go index 9b62231eb..2f1e119e6 100644 --- a/services/core/internal/runtimeobs/sampler_test.go +++ b/services/core/internal/runtimeobs/sampler_test.go @@ -125,13 +125,13 @@ func TestSamplerSweepsEveryPageAndIsolatesSessionFailures(t *testing.T) { "session-b": {Sessions: []SessionIdentity{{TenantID: "tenant-c", SessionID: "session-c"}}}, }} observer := &samplerObserver{fail: map[string]bool{"session-b": true}} - sampler, err := NewSampler(lister, observer, samplerOwner{}, SamplerOptions{Interval: time.Minute, PageSize: 2, Concurrency: 2}) + sampler, err := NewSampler(lister, observer, samplerOwner{}, SamplerOptions{PageSize: 2, Concurrency: 2}) if err != nil { t.Fatal(err) } now := time.Date(2026, 9, 23, 4, 0, 0, 0, time.UTC) sampler.now = func() time.Time { now = now.Add(time.Second); return now } - result := sampler.sweep(t.Context()) + result := sampler.Sweep(t.Context()) if !result.Complete || result.Listed != 3 || result.Observed != 2 || result.Failed != 1 { t.Fatalf("unexpected sweep result: %+v", result) } @@ -155,11 +155,11 @@ func TestSamplerBoundsConcurrencyAndSourceDeadline(t *testing.T) { {TenantID: "t", SessionID: "1"}, {TenantID: "t", SessionID: "2"}, {TenantID: "t", SessionID: "3"}, }}, }} - sampler, err := NewSampler(lister, service, samplerOwner{}, SamplerOptions{Interval: time.Minute, Concurrency: 2, SourceTimeout: 20 * time.Millisecond}) + sampler, err := NewSampler(lister, service, samplerOwner{}, SamplerOptions{Concurrency: 2, SourceTimeout: 20 * time.Millisecond}) if err != nil { t.Fatal(err) } - result := sampler.sweep(t.Context()) + result := sampler.Sweep(t.Context()) // Source deadlines are recorded as sample_timeout observations. if !result.Complete || result.Observed != 3 || result.Failed != 0 || source.max != 2 { t.Fatalf("unexpected bounded result: result=%+v max=%d", result, source.max) @@ -168,11 +168,11 @@ func TestSamplerBoundsConcurrencyAndSourceDeadline(t *testing.T) { func TestSamplerStopsBeforeListingWithoutOwnership(t *testing.T) { lister := &samplerLister{pages: map[string]SessionPage{}} - sampler, err := NewSampler(lister, &samplerObserver{}, samplerOwner{err: errors.New("lost")}, SamplerOptions{Interval: time.Minute}) + sampler, err := NewSampler(lister, &samplerObserver{}, samplerOwner{err: errors.New("lost")}, SamplerOptions{}) if err != nil { t.Fatal(err) } - result := sampler.sweep(t.Context()) + result := sampler.Sweep(t.Context()) if result.Complete || len(lister.cursors) != 0 { t.Fatalf("sampler ran without deployment ownership: %+v %#v", result, lister.cursors) } @@ -184,83 +184,27 @@ func TestSamplerRejectsInvalidContinuationWithoutLooping(t *testing.T) { NextCursor: "different-session", }}} observer := &samplerObserver{} - sampler, err := NewSampler(lister, observer, samplerOwner{}, SamplerOptions{Interval: time.Minute}) + sampler, err := NewSampler(lister, observer, samplerOwner{}, SamplerOptions{}) if err != nil { t.Fatal(err) } - result := sampler.sweep(t.Context()) + result := sampler.Sweep(t.Context()) if result.Complete || result.Listed != 0 || len(observer.sessions) != 0 || len(lister.cursors) != 1 { t.Fatalf("invalid continuation was accepted: result=%+v sessions=%#v cursors=%#v", result, observer.sessions, lister.cursors) } } -func TestSamplerReportPanicIsIsolated(t *testing.T) { - sampler, err := NewSampler( - &samplerLister{pages: map[string]SessionPage{}}, - &samplerObserver{}, - samplerOwner{}, - SamplerOptions{Interval: time.Minute, Report: func(SweepResult) { panic("test") }}, - ) - if err != nil { - t.Fatal(err) - } - sampler.report(SweepResult{Complete: true}) -} - -func TestSamplerRunDoesNotOverlapSweepsAndStops(t *testing.T) { - started := make(chan struct{}, 1) - release := make(chan struct{}) - observer := &samplerObserver{wait: release} - lister := &samplerLister{pages: map[string]SessionPage{"": {Sessions: []SessionIdentity{{TenantID: "t", SessionID: "s"}}}}} - sampler, err := NewSampler(lister, observer, samplerOwner{}, SamplerOptions{Interval: time.Millisecond, SourceTimeout: time.Second}) - if err != nil { - t.Fatal(err) - } - sampler.afterSweep = func(SweepResult) { started <- struct{}{} } - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error, 1) - go func() { done <- sampler.Run(ctx) }() - deadline := time.After(time.Second) - for { - observer.mu.Lock() - active := observer.active - max := observer.max - observer.mu.Unlock() - if active == 1 { - if max != 1 { - t.Fatalf("overlapping sweep observed: max=%d", max) - } - break - } - select { - case <-deadline: - t.Fatal("sampler did not start") - default: - time.Sleep(time.Millisecond) - } - } - cancel() - select { - case err := <-done: - if !errors.Is(err, context.Canceled) { - t.Fatalf("unexpected sampler exit: %v", err) - } - case <-time.After(time.Second): - t.Fatal("sampler did not stop") - } -} - func TestSamplerCancelsProviderReadWhenOwnershipIsLost(t *testing.T) { release := make(chan struct{}) observer := &samplerObserver{wait: release} owner := &sequenceOwner{} lister := &samplerLister{pages: map[string]SessionPage{"": {Sessions: []SessionIdentity{{TenantID: "t", SessionID: "s"}}}}} - sampler, err := NewSampler(lister, observer, owner, SamplerOptions{Interval: time.Minute, SourceTimeout: time.Second}) + sampler, err := NewSampler(lister, observer, owner, SamplerOptions{SourceTimeout: time.Second}) if err != nil { t.Fatal(err) } done := make(chan SweepResult, 1) - go func() { done <- sampler.sweep(t.Context()) }() + go func() { done <- sampler.Sweep(t.Context()) }() deadline := time.After(time.Second) for { observer.mu.Lock() @@ -310,12 +254,12 @@ func TestSamplerPreservesProviderTimeoutAndFinalFenceAfterSlowResolution(t *test } owner := &sequenceOwner{} sampler, err := NewSampler(resolver, service, owner, SamplerOptions{ - Interval: time.Minute, SourceTimeout: 10 * time.Millisecond, + SourceTimeout: 10 * time.Millisecond, }) if err != nil { t.Fatal(err) } - result := sampler.sweep(t.Context()) + result := sampler.Sweep(t.Context()) if !result.Complete || result.Observed != 1 || result.Failed != 0 { t.Fatalf("slow-resolution sweep lost the timeout observation: %+v", result) } diff --git a/services/web/Dockerfile b/services/web/Dockerfile index cea7c0df0..ec2e587fd 100644 --- a/services/web/Dockerfile +++ b/services/web/Dockerfile @@ -3,7 +3,6 @@ FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55 COPY --chmod=0555 oac-web /usr/local/bin/oac-web COPY dist /www -ENV OAC_WEB_ADDR=:8080 OAC_WEB_DIST=/www EXPOSE 8080 USER 65532:65532 CMD ["/usr/local/bin/oac-web"] diff --git a/services/web/config.go b/services/web/config.go index cd2f2413f..9d64eff35 100644 --- a/services/web/config.go +++ b/services/web/config.go @@ -1,6 +1,7 @@ package main import ( + "cmp" "errors" "io" "net/url" @@ -9,32 +10,41 @@ import ( "strings" "unicode" "unicode/utf8" + + "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) type config struct { addr, origin, dist string coreKey, nodePayloadDir string upstream *url.URL + log log.Config } +// loadConfig reads Web's settings. An unset or empty variable selects its +// default; OAC_PUBLIC_URL and OAC_WEB_CORE_KEY_FILE have none. func loadConfig() (config, error) { c := config{ - addr: envDefault("OAC_WEB_ADDR", ":8080"), - origin: envDefault("OAC_WEB_ORIGIN", "http://127.0.0.1:8080"), - dist: envDefault("OAC_WEB_DIST", "/www"), + addr: cmp.Or(os.Getenv("OAC_WEB_ADDR"), ":8080"), + origin: os.Getenv("OAC_PUBLIC_URL"), + dist: cmp.Or(os.Getenv("OAC_WEB_DIST"), "/www"), + } + var err error + if c.log, err = log.LoadConfig(); err != nil { + return config{}, err } origin, err := serverURL(c.origin) if err != nil || origin.Path != "" { - return config{}, errors.New("OAC_WEB_ORIGIN must be an HTTP(S) origin without a path") + return config{}, errors.New("OAC_PUBLIC_URL must be an HTTP(S) origin without a path") } - c.upstream, err = serverURL(envDefault("OAC_WEB_UPSTREAM", "http://core:8091")) + c.upstream, err = serverURL(cmp.Or(os.Getenv("OAC_WEB_UPSTREAM"), "http://core:8091")) if err != nil { return config{}, errors.New("OAC_WEB_UPSTREAM must be an HTTP(S) server URL without credentials, query or path") } if !filepath.IsAbs(c.dist) { return config{}, errors.New("OAC_WEB_DIST must be absolute") } - c.coreKey, err = readSecret(envDefault("OAC_WEB_CORE_KEY_FILE", "/admin/core.key")) + c.coreKey, err = readSecret(os.Getenv("OAC_WEB_CORE_KEY_FILE")) if err != nil { return config{}, errors.New("OAC_WEB_CORE_KEY_FILE must name a private regular file containing the Core key") } @@ -48,13 +58,6 @@ func loadConfig() (config, error) { return c, nil } -func envDefault(key, fallback string) string { - if value, exists := os.LookupEnv(key); exists { - return value - } - return fallback -} - func serverURL(value string) (*url.URL, error) { u, err := url.Parse(value) if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || diff --git a/services/web/config_test.go b/services/web/config_test.go index ae1e3ccac..978efebe0 100644 --- a/services/web/config_test.go +++ b/services/web/config_test.go @@ -15,14 +15,14 @@ func TestConfigRejectsUnsafeURLsAndSecretFiles(t *testing.T) { t.Fatal(err) } t.Setenv("OAC_WEB_CORE_KEY_FILE", key) - t.Setenv("OAC_WEB_ORIGIN", testOrigin) + t.Setenv("OAC_PUBLIC_URL", testOrigin) t.Setenv("OAC_WEB_UPSTREAM", "http://core:8091") t.Setenv("OAC_WEB_DIST", directory) c, err := loadConfig() if err != nil || c.coreKey != valid { t.Fatalf("valid configuration failed: %v", err) } - for _, value := range []string{"http://user:secret@core:8091", "http://core:8091/v1", "http://core:8091?token=secret", "http://core:8091#", "file:///config/caller.key", ""} { + for _, value := range []string{"http://user:secret@core:8091", "http://core:8091/v1", "http://core:8091?token=secret", "http://core:8091#", "file:///config/caller.key"} { t.Run(value, func(t *testing.T) { t.Setenv("OAC_WEB_UPSTREAM", value) _, err := loadConfig() @@ -31,6 +31,15 @@ func TestConfigRejectsUnsafeURLsAndSecretFiles(t *testing.T) { } }) } + t.Setenv("OAC_WEB_UPSTREAM", "") + if c, err := loadConfig(); err != nil || c.upstream.String() != "http://core:8091" { + t.Fatal("an empty upstream did not select the default", err) + } + t.Setenv("OAC_PUBLIC_URL", "") + if _, err := loadConfig(); err == nil || !strings.Contains(err.Error(), "OAC_PUBLIC_URL") { + t.Fatal("console configured without OAC_PUBLIC_URL", err) + } + t.Setenv("OAC_PUBLIC_URL", testOrigin) for _, value := range []string{"", "token with spaces", strings.Repeat("x", 4097), "token\x00", strings.Repeat("s", 31)} { if err := os.WriteFile(key, []byte(value), 0o600); err != nil { t.Fatal(err) @@ -58,7 +67,7 @@ func TestBootstrapFollowsManagedHTTPOrigin(t *testing.T) { } t.Setenv("OAC_WEB_CORE_KEY_FILE", key) t.Setenv("OAC_WEB_DIST", directory) - t.Setenv("OAC_WEB_ORIGIN", "http://localhost:8080") + t.Setenv("OAC_PUBLIC_URL", "http://localhost:8080") if _, err := loadConfig(); err != nil { t.Fatal(err) } diff --git a/services/web/main.go b/services/web/main.go index a2459d7a4..3bc10bee6 100644 --- a/services/web/main.go +++ b/services/web/main.go @@ -5,7 +5,9 @@ import ( "context" "errors" "fmt" + "net" "net/http" + "net/url" "os" "os/signal" "syscall" @@ -38,22 +40,33 @@ func main() { // printCoreKey writes the sign-in key for `docker compose exec web oac-web // core-key`. Exec output never enters the container log. func printCoreKey() error { - key, err := readSecret(envDefault("OAC_WEB_CORE_KEY_FILE", "/admin/core.key")) + c, err := loadConfig() if err != nil { - return errors.New("cannot read the Core key from OAC_WEB_CORE_KEY_FILE") + return err } - fmt.Println(key) + fmt.Println(c.coreKey) return nil } // healthcheck reports the installation healthy once Core and Web's own listener // answer. Compose runs it inside the web container. func healthcheck() error { + c, err := loadConfig() + if err != nil { + return err + } + host, port, err := net.SplitHostPort(c.addr) + if err != nil { + return errors.New("OAC_WEB_ADDR must be a host:port listen address") + } + if ip := net.ParseIP(host); host == "" || (ip != nil && ip.IsUnspecified()) { + host = "127.0.0.1" + } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{Proxy: nil}} - for _, host := range []string{"core:8091", "127.0.0.1:8080"} { - request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+host+"/healthz", nil) + for _, server := range []*url.URL{c.upstream, {Scheme: "http", Host: net.JoinHostPort(host, port)}} { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, server.JoinPath("healthz").String(), nil) if err != nil { return err } @@ -63,18 +76,18 @@ func healthcheck() error { } response.Body.Close() if response.StatusCode != http.StatusOK { - return fmt.Errorf("%s returned HTTP %d", host, response.StatusCode) + return fmt.Errorf("%s returned HTTP %d", server.Host, response.StatusCode) } } return nil } func run() error { - log.Init(log.ConfigFromEnv()) c, err := loadConfig() if err != nil { return err } + log.Init(c.log) handler, err := newConsole(c) if err != nil { return err