diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 9f9350f..c609af5 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "jev-code", "displayName": "Jev for coding agents", - "version": "0.4.1", + "version": "0.5.0", "description": "Jev, TypeSafe's System One classifier, as tools (jev_classify, jev_check, jev_score, jev_rank, jev_ask) plus a skill that says when to use them.", "author": { "name": "Fran\u00e7ois Chastel", diff --git a/.env.example b/.env.example index 4b166fc..0d53ab0 100644 --- a/.env.example +++ b/.env.example @@ -5,14 +5,14 @@ TYPESAFE_API_KEY= # OPENROUTER_API_KEY= # Vercel AI Gateway: https://vercel.com/docs/ai-gateway/authentication-and-byok/api-keys # AI_GATEWAY_API_KEY= -# OpenAI Decisions API (limited preview); opt in with JEV_CODE_PROVIDER=openai +# OpenAI Decisions API (public beta); opt in with JEV_CODE_PROVIDER=openai # OPENAI_API_KEY= # Ollama (local, no key); opt in with JEV_CODE_PROVIDER=ollama after `ollama pull nimble` # OLLAMA_HOST=localhost:11434 # Optional overrides (defaults shown) # JEV_CODE_PROVIDER=typesafe # typesafe | openrouter | vercel | openai | ollama; forces the host (required for the last two) -# TYPESAFE_BASE_URL=https://api.typesafe.ai +# TYPESAFE_BASE_URL=https://api.typesafe.ai # a proxy or a System One gateway of your own; /v1/systemone is appended # TYPESAFE_DEFAULT_MODEL=jev-latest # typesafe-ai/jev on Vercel AI Gateway # JEV_CODE_TIMEOUT_MS=30000 # JEV_CODE_MAX_RETRIES=2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5586f53..ca21eea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,3 +53,13 @@ jobs: else npm run test:e2e fi + - name: Run live tests against OpenAI's Decisions API when a key is configured + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + JEV_CODE_PROVIDER: openai + run: | + if [ -z "$OPENAI_API_KEY" ]; then + echo "No OPENAI_API_KEY secret set; skipping the OpenAI live tests." + else + npm run test:e2e + fi diff --git a/.mcp.json b/.mcp.json index 40ba31c..40cd23c 100644 --- a/.mcp.json +++ b/.mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "jev": { "command": "npx", - "args": ["-y", "@french-castle/jev-code@0.4.1", "mcp"] + "args": ["-y", "@french-castle/jev-code@0.5.0", "mcp"] } } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 5505c89..b19aae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ All notable changes to this project are documented here. The format follows ## [Unreleased] +## [0.5.0] - 2026-10-07 + +OpenAI's Decisions API, now in public beta for every developer, and a System One gateway of your +own as a first-class setup path. One change to check before upgrading: `TYPESAFE_BASE_URL` is now +validated, so a value carrying credentials, a query string, or a trailing `/v1/systemone` is +refused before any request instead of being sent. + +### Added + +- A System One gateway of your own, documented as what it already was: the proxy case. The key + goes in `TYPESAFE_API_KEY`, the SDK's generic variable, and the address in `TYPESAFE_BASE_URL`, + with `TYPESAFE_DEFAULT_MODEL` when the gateway's model id differs; nothing new to configure in + any harness. `setup` offers "Other System One gateway" when asking which host a pasted key is + for, then asks for the URL and the model and stores the three variables together. +- OpenAI's Decisions API in its public beta, open to every developer since October 6, 2026. + The mapping now follows OpenAI's published reference instead of recorded preview traffic; the + request jev-code sent already matched it, so existing setups keep working unchanged. All five + tools and the live test suite were run against `gpt-6-luna` before this release. +- A question the model declines comes back as a refusal on the Decisions API. Tools report it as + `status: "refused"` with the cautious outcome (`review`, or `uncertain` in `jev_check`) + instead of `invalid_response`, since retrying will not change it, and `jev_ask` lists declined + ids under `refused`. +- CI runs the live tests against the Decisions API too when an `OPENAI_API_KEY` secret is set. + +### Changed + +- `TYPESAFE_BASE_URL` is checked before any request: an absolute `http(s)` URL with no + credentials, query, or fragment, stopping before `/v1/systemone`, which the client appends. + A bad value is a configuration problem in `doctor` and `setup`, never echoed in case it embeds + a secret, instead of a connection error at the first call. +- Score probabilities from the Decisions API are keyed by the level index the API returns, + falling back to the label. +- `doctor --live` no longer calls a `403` from the Decisions API a preview restriction; it + quotes OpenAI's message and points at the key's project access to `gpt-6-luna`. + +### Fixed + +- `npm run test:e2e` found no tests to run, because the test config excluded the live suite for + every run, so the CI live job could never have exercised a host. The suite is now excluded + only when `JEV_CODE_E2E` is unset. + ## [0.4.1] - 2026-10-03 ### Fixed @@ -136,7 +177,8 @@ each harness, and skill discovery. The first version published to npm. - CLI access to every tool (`jev-code classify --input payload.json`). - Claude Code plugin manifest and marketplace so the repository installs as a plugin. -[Unreleased]: https://github.com/FrancoisChastel/jev-code/compare/v0.4.1...HEAD +[Unreleased]: https://github.com/FrancoisChastel/jev-code/compare/v0.5.0...HEAD +[0.5.0]: https://github.com/FrancoisChastel/jev-code/compare/v0.4.1...v0.5.0 [0.4.1]: https://github.com/FrancoisChastel/jev-code/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/FrancoisChastel/jev-code/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/FrancoisChastel/jev-code/compare/v0.2.1...v0.3.0 diff --git a/README.md b/README.md index 4555c91..d97961a 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ npx -y @french-castle/jev-code@latest setup That detects the harnesses on your machine and, for each one, copies the skill and registers the tool. When no API key is in your shell, it asks for one right there (hidden input) and which host -it belongs to: TypeSafe, OpenRouter, Vercel AI Gateway, or OpenAI. There is nothing else to -configure. +it belongs to: TypeSafe, OpenRouter, Vercel AI Gateway, OpenAI, or a System One gateway of your +own, in which case it also asks for the gateway's URL. There is nothing else to configure. Add harness names to be explicit (`setup claude codex pi opencode`), `--project` to install into the current repository instead of your user profile, or `--dry-run` to see the plan first. @@ -139,7 +139,7 @@ override per call. Policy stays in your hands; Jev supplies the probabilities. Claude Code `jev-code setup claude` copies the skill to `~/.claude/skills/jev/` and runs -`claude mcp add --scope user jev -- npx -y @french-castle/jev-code@0.4.1 mcp`. The tools appear as +`claude mcp add --scope user jev -- npx -y @french-castle/jev-code@0.5.0 mcp`. The tools appear as `mcp__jev__jev_classify` and friends; the skill is `/jev`. Prefer a plugin that updates itself? This repository is also a Claude Code plugin marketplace: @@ -158,7 +158,7 @@ project-scope notes: [docs/harnesses/claude-code.md](docs/harnesses/claude-code. Codex `jev-code setup codex` copies the skill to `~/.agents/skills/jev/` (Codex's user-level skills -directory, shared with Pi and OpenCode) and runs `codex mcp add jev -- npx -y @french-castle/jev-code@0.4.1 mcp`. +directory, shared with Pi and OpenCode) and runs `codex mcp add jev -- npx -y @french-castle/jev-code@0.5.0 mcp`. Without the `codex` binary it appends a `[mcp_servers.jev]` table to `~/.codex/config.toml` instead. Invoke the skill with `$jev`. Details: [docs/harnesses/codex.md](docs/harnesses/codex.md). @@ -194,7 +194,7 @@ works for the 70+ agents it supports: npx skills add FrancoisChastel/jev-code --skill jev ``` -Pair it with the MCP server (`npx -y @french-castle/jev-code@0.4.1 mcp`) in your agent's MCP config, +Pair it with the MCP server (`npx -y @french-castle/jev-code@0.5.0 mcp`) in your agent's MCP config, or let the agent fall back to the CLI. @@ -215,7 +215,7 @@ Output is JSON on stdout. Exit code 2 means a usage or configuration problem, 1 ## Upgrading -Harness configs launch a pinned command, `npx -y @french-castle/jev-code@0.4.1 mcp`, so an agent +Harness configs launch a pinned command, `npx -y @french-castle/jev-code@0.5.0 mcp`, so an agent starts fast and works offline once that version is cached, and nothing changes under you until you decide. To upgrade, run setup again with `@latest`: it re-registers the tool with the new pin, replacing the old entry (also through `claude mcp` and `codex mcp`), and refreshes the copied @@ -232,16 +232,17 @@ on their own. Run the command above once to move them to a pinned command. ## Configuration One host is required. The first three below serve Jev behind the same System One API, so the -tools behave identically and only the account you pay through changes. The last two are other -models that answer the same kinds of questions: OpenAI's Decisions API, and Ollama's local -decision models, which need no key at all. +tools behave identically and only the account you pay through changes; so does a gateway of your +own that speaks that API ([below](#your-own-gateway)). The last two are other models that answer +the same kinds of questions: OpenAI's Decisions API, and Ollama's local decision models, which +need no key at all. | Host | Key variable | Default model | Get a key | | --- | --- | --- | --- | | TypeSafe (direct) | `TYPESAFE_API_KEY` | `jev-latest` | [console.typesafe.ai/keys](https://console.typesafe.ai/keys) | | OpenRouter | `OPENROUTER_API_KEY` | `jev-latest` | [openrouter.ai/settings/keys](https://openrouter.ai/settings/keys) | | Vercel AI Gateway | `AI_GATEWAY_API_KEY` | `typesafe-ai/jev` | [AI Gateway API keys](https://vercel.com/docs/ai-gateway/authentication-and-byok/api-keys) | -| OpenAI Decisions API (preview, opt-in) | `OPENAI_API_KEY` + `JEV_CODE_PROVIDER=openai` | `gpt-6-luna` | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | +| OpenAI Decisions API (public beta, opt-in) | `OPENAI_API_KEY` + `JEV_CODE_PROVIDER=openai` | `gpt-6-luna` | [platform.openai.com/api-keys](https://platform.openai.com/api-keys) | | Ollama (local, opt-in) | no key; `JEV_CODE_PROVIDER=ollama`, `OLLAMA_HOST` optional | `nimble` | [docs.ollama.com/capabilities/decision](https://docs.ollama.com/capabilities/decision) | How the host is chosen: @@ -265,27 +266,35 @@ How the host is chosen: | --- | --- | --- | | `JEV_CODE_PROVIDER` | auto | Force `typesafe`, `openrouter`, `vercel`, `openai`, or `ollama` (required for the last two). | | `OLLAMA_HOST` | `localhost:11434` | Ollama's server, as a URL, `host:port`, or bare host. | -| `TYPESAFE_BASE_URL` | per host | Point at a proxy or a compatible endpoint. TypeSafe keys only, unless `JEV_CODE_PROVIDER` is set. | +| `TYPESAFE_BASE_URL` | per host | A proxy, or a System One gateway of your own; the client appends `/v1/systemone`. TypeSafe keys only, unless `JEV_CODE_PROVIDER` is set. | | `TYPESAFE_DEFAULT_MODEL` | per host | Pin a Jev version: `jev-1.13` on TypeSafe or OpenRouter; Vercel uses `typesafe-ai/jev`. | | `JEV_CODE_TIMEOUT_MS` | `30000` | Per-attempt timeout. | | `JEV_CODE_MAX_RETRIES` | `2` | Retries on 429, 5xx, timeouts, and connection errors. | The `TYPESAFE_*` names match the official TypeSafe SDKs, so one export serves everything. -### OpenAI Decisions API (preview) +### OpenAI Decisions API (public beta) -OpenAI announced a Decisions API at DevDay 2026 that answers the same three question shapes -(yes/no, pick one, score on levels) with probabilities, served by GPT-6 Luna. It is in limited -preview with no public reference yet; accounts outside the preview get `403 Decision API is not -enabled for this user`, which `doctor --live` reports in plain words. jev-code maps every tool -onto it, so the payloads and results are unchanged, with these differences: +OpenAI's Decisions API answers the same three question shapes (yes/no, pick one, score on +levels) with probabilities, served by GPT-6 Luna. It opened to every developer in public beta on +October 6, 2026, with general availability expected in the following weeks; see OpenAI's +[guide](https://developers.openai.com/api/docs/guides/decisions). jev-code maps every tool onto +it, so the payloads and results are unchanged, with these differences: -- The request shape differs from System One. jev-code translates it; the mapping follows traffic - recorded by preview users and OpenAI's own client in the Codex repository, not a published - spec, and was not verified against a live preview account. +- The request shape differs from System One. jev-code translates it, following OpenAI's + [API reference](https://developers.openai.com/api/reference/resources/decisions/methods/create). - Yes/no questions have no separate criteria field, so `yes` and `no` descriptions are folded into the instructions. Score legends are rebuilt from your levels. -- Pricing and limits are OpenAI's and not published. Tool outputs still carry `usage`. +- The model can decline a question while answering the others. That item comes back with + `status: "refused"` and a `review` decision (an `uncertain` verdict in `jev_check`), and + `jev_ask` lists its id under `refused`. Retrying the same input will not change it. +- The API also takes images; the tools stay text-only, as they do on every host. +- Pricing is OpenAI's: at launch, $0.10 per million input tokens and nothing for output. Tool + outputs carry `usage`. + +It stays opt-in. `OPENAI_API_KEY` is set in many shells for other reasons, so sending agent +evidence to OpenAI takes `JEV_CODE_PROVIDER=openai` (or pasting the key when `setup` asks). A +`403` from the API is reported in plain words by `doctor --live`. ### Ollama (local) @@ -303,6 +312,27 @@ picks one). `OLLAMA_HOST` points at a server other than `localhost:11434`, in Ol notation. Image inputs, which `clef` supports, are not exposed by the tools. `doctor --live` says when the server is down or the model is not pulled. +### Your own gateway + +Any server that speaks the System One API (`POST /v1/systemone`, same request and response +shapes) works without a row in the table: it is the proxy case the TypeSafe SDK already knows. +The gateway's key goes in `TYPESAFE_API_KEY`, the SDK's generic variable, and its address in +`TYPESAFE_BASE_URL`; `TYPESAFE_DEFAULT_MODEL` names the model when the gateway does not serve +`jev-latest`: + +```bash +export TYPESAFE_API_KEY=... # the gateway's key +export TYPESAFE_BASE_URL=https://gateway.example/api # the client appends /v1/systemone +export TYPESAFE_DEFAULT_MODEL=vendor/jev # when the gateway's model id differs +``` + +`setup` offers the same thing when it asks which host a pasted key is for: choose "Other System +One gateway", give the URL and the model, and the three variables go where the key would have +gone. The URL is checked before anything is sent: an absolute `http(s)` URL with no credentials, +query, or fragment, stopping before `/v1/systemone`. A host-specific key (`OPENROUTER_API_KEY` +and the like) never follows `TYPESAFE_BASE_URL` to a gateway unless `JEV_CODE_PROVIDER` names +its host; `doctor` shows where requests go. + ## Security notes - Only the payload you pass to a tool leaves your machine, and only to the host your key diff --git a/SECURITY.md b/SECURITY.md index b87c16b..b6c1734 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,8 +19,10 @@ One `POST .../v1/systemone` per tool call, to the host your key belongs to: - `https://api.openai.com` (OpenAI Decisions API), only when `JEV_CODE_PROVIDER=openai` - `http://localhost:11434`, or `OLLAMA_HOST` (Ollama), only when `JEV_CODE_PROVIDER=ollama`; no key is sent unless `OLLAMA_API_KEY` is set -- or `TYPESAFE_BASE_URL` when set, under the proxy rule below. A base URL on one of the known - hosts above while `JEV_CODE_PROVIDER` names another is refused, for Ollama too. +- or `TYPESAFE_BASE_URL` when set, under the proxy rule below: a proxy, or a System One gateway + of your own. The value is checked before any request (absolute `http(s)`, no embedded + credentials, query, or fragment). A base URL on one of the known hosts above while + `JEV_CODE_PROVIDER` names another is refused, for Ollama too. The payload is what the agent passed to a tool: items, questions, optional context. Through OpenRouter or Vercel AI Gateway it transits that gateway on its way to TypeSafe. jev-code never diff --git a/docs/harnesses/claude-code.md b/docs/harnesses/claude-code.md index 200987c..b822be4 100644 --- a/docs/harnesses/claude-code.md +++ b/docs/harnesses/claude-code.md @@ -12,7 +12,7 @@ What it does: | Piece | User scope | Project scope | | --- | --- | --- | | Skill | `~/.claude/skills/jev/` | `.claude/skills/jev/` | -| Tool | `claude mcp add --scope user jev -- npx -y @french-castle/jev-code@0.4.1 mcp` | `claude mcp add --scope project ...`, which writes `.mcp.json` | +| Tool | `claude mcp add --scope user jev -- npx -y @french-castle/jev-code@0.5.0 mcp` | `claude mcp add --scope project ...`, which writes `.mcp.json` | If the `claude` binary is not on PATH, project scope writes `.mcp.json` directly and user scope prints the command to run. The command pins the version setup installed; re-running setup @@ -36,7 +36,7 @@ The skill is then invoked as `/jev-code:jev`, and the tools appear as Register the server yourself: ```bash -claude mcp add --scope user jev -e TYPESAFE_API_KEY= -- npx -y @french-castle/jev-code@0.4.1 mcp +claude mcp add --scope user jev -e TYPESAFE_API_KEY= -- npx -y @french-castle/jev-code@0.5.0 mcp ``` or add it to a project's `.mcp.json`: @@ -46,7 +46,7 @@ or add it to a project's `.mcp.json`: "mcpServers": { "jev": { "command": "npx", - "args": ["-y", "@french-castle/jev-code@0.4.1", "mcp"], + "args": ["-y", "@french-castle/jev-code@0.5.0", "mcp"], "env": { "TYPESAFE_API_KEY": "" } } } @@ -55,8 +55,10 @@ or add it to a project's `.mcp.json`: Any provider key works in place of `TYPESAFE_API_KEY`: `OPENROUTER_API_KEY` routes through OpenRouter and `AI_GATEWAY_API_KEY` through Vercel AI Gateway (the variable decides the host). -OpenAI's Decisions API (preview) needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the -environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. +OpenAI's Decisions API needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the +environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. A System One gateway of your +own takes `TYPESAFE_API_KEY` plus `TYPESAFE_BASE_URL`, as the +[README](../../README.md#your-own-gateway) explains. Install the skill with `npx skills add FrancoisChastel/jev-code --skill jev -a claude-code`, or copy `skills/jev/` into `~/.claude/skills/`. diff --git a/docs/harnesses/codex.md b/docs/harnesses/codex.md index 9f65fef..068032b 100644 --- a/docs/harnesses/codex.md +++ b/docs/harnesses/codex.md @@ -11,7 +11,7 @@ What it does: | Piece | Location | | --- | --- | | Skill | `~/.agents/skills/jev/` (Codex's user-level skills directory; Pi and OpenCode read it too). With `--project`: `.agents/skills/jev/`. | -| Tool | `codex mcp add jev -- npx -y @french-castle/jev-code@0.4.1 mcp`, or a `[mcp_servers.jev]` table appended to `~/.codex/config.toml` when the `codex` binary is not on PATH. | +| Tool | `codex mcp add jev -- npx -y @french-castle/jev-code@0.5.0 mcp`, or a `[mcp_servers.jev]` table appended to `~/.codex/config.toml` when the `codex` binary is not on PATH. | Codex keeps MCP servers in its user configuration, so `--project` still registers the tool at user level and only the skill moves into the repository. @@ -27,7 +27,7 @@ Add to `~/.codex/config.toml`: ```toml [mcp_servers.jev] command = "npx" -args = ["-y", "@french-castle/jev-code@0.4.1", "mcp"] +args = ["-y", "@french-castle/jev-code@0.5.0", "mcp"] [mcp_servers.jev.env] TYPESAFE_API_KEY = "" @@ -35,8 +35,10 @@ TYPESAFE_API_KEY = "" Any provider key works in place of `TYPESAFE_API_KEY`: `OPENROUTER_API_KEY` routes through OpenRouter and `AI_GATEWAY_API_KEY` through Vercel AI Gateway (the variable decides the host). -OpenAI's Decisions API (preview) needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the -environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. +OpenAI's Decisions API needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the +environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. A System One gateway of your +own takes `TYPESAFE_API_KEY` plus `TYPESAFE_BASE_URL`, as the +[README](../../README.md#your-own-gateway) explains. Install the skill with `npx skills add FrancoisChastel/jev-code --skill jev -a codex`, or copy `skills/jev/` into `~/.agents/skills/`. diff --git a/docs/harnesses/opencode.md b/docs/harnesses/opencode.md index 3f1a716..37039df 100644 --- a/docs/harnesses/opencode.md +++ b/docs/harnesses/opencode.md @@ -28,7 +28,7 @@ Add to `opencode.json`: "mcp": { "jev": { "type": "local", - "command": ["npx", "-y", "@french-castle/jev-code@0.4.1", "mcp"], + "command": ["npx", "-y", "@french-castle/jev-code@0.5.0", "mcp"], "enabled": true, "environment": { "TYPESAFE_API_KEY": "" } } @@ -38,8 +38,10 @@ Add to `opencode.json`: Any provider key works in place of `TYPESAFE_API_KEY`: `OPENROUTER_API_KEY` routes through OpenRouter and `AI_GATEWAY_API_KEY` through Vercel AI Gateway (the variable decides the host). -OpenAI's Decisions API (preview) needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the -environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. +OpenAI's Decisions API needs both `OPENAI_API_KEY` and `JEV_CODE_PROVIDER=openai` in the +environment; a local Ollama needs only `JEV_CODE_PROVIDER=ollama`. A System One gateway of your +own takes `TYPESAFE_API_KEY` plus `TYPESAFE_BASE_URL`, as the +[README](../../README.md#your-own-gateway) explains. Install the skill with `npx skills add FrancoisChastel/jev-code --skill jev -a opencode`, or copy `skills/jev/` into `~/.config/opencode/skills/`. diff --git a/docs/harnesses/pi.md b/docs/harnesses/pi.md index 4e90640..26608de 100644 --- a/docs/harnesses/pi.md +++ b/docs/harnesses/pi.md @@ -29,8 +29,9 @@ project; `setup --project` passes it. The extension reads the API key (`TYPESAFE_API_KEY`, `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`; OpenAI's Decisions API with `OPENAI_API_KEY` plus `JEV_CODE_PROVIDER=openai`; -a local Ollama with just `JEV_CODE_PROVIDER=ollama`) from the shell pi runs in; nothing is written -into pi's settings. +a local Ollama with just `JEV_CODE_PROVIDER=ollama`; a System One gateway of your own with +`TYPESAFE_API_KEY` plus `TYPESAFE_BASE_URL`) from the shell pi runs in; nothing is written into +pi's settings. ## Verify diff --git a/package-lock.json b/package-lock.json index 5a4cfc5..c8199d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@french-castle/jev-code", - "version": "0.4.1", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@french-castle/jev-code", - "version": "0.4.1", + "version": "0.5.0", "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.30.1", diff --git a/package.json b/package.json index f98c771..92193a8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@french-castle/jev-code", - "version": "0.4.1", + "version": "0.5.0", "description": "Jev, TypeSafe's System One classifier, as a tool inside Claude Code, Codex, Pi, and OpenCode: typed classify, check, score, rank, and ask, plus one-command setup.", "type": "module", "license": "MIT", diff --git a/skills/jev/SKILL.md b/skills/jev/SKILL.md index a4c7f86..d4c1562 100644 --- a/skills/jev/SKILL.md +++ b/skills/jev/SKILL.md @@ -16,7 +16,7 @@ compatibility: Requires Node.js 20+ and one API key in the environment (TYPESAFE metadata: author: FrancoisChastel source: https://github.com/FrancoisChastel/jev-code - version: "0.4.1" + version: "0.5.0" --- # Jev: a classifier for coding agents @@ -126,8 +126,9 @@ cookbook links, judgment design, and SDK snippets, adapted from TypeSafe's own s Keep destructive actions behind your own confirmation, whatever the confidence. - The API key is read from `TYPESAFE_API_KEY`, `OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY` (OpenAI's Decisions API only with `JEV_CODE_PROVIDER=openai`; a local Ollama with - `JEV_CODE_PROVIDER=ollama` and no key); `jev-code doctor` shows which host is in use. Never - print a key, and never paste one into chat. + `JEV_CODE_PROVIDER=ollama` and no key; a System One gateway of the user's own with + `TYPESAFE_API_KEY` plus `TYPESAFE_BASE_URL`); `jev-code doctor` shows which host is in use. + Never print a key, and never paste one into chat. ## Troubleshooting @@ -137,4 +138,5 @@ cookbook links, judgment design, and SDK snippets, adapted from TypeSafe's own s | "Request is N characters, above the budget" | Split items into batches, or shorten texts to the deciding excerpt. | | Many `review` results | Sharpen class descriptions, add a catch-all, or pass more context. | | `status: invalid_response` on an item | The API answered in an unexpected shape; retry once, then report it. | +| `status: refused` on an item | The host declined that question (OpenAI's Decisions API can). Retrying will not help; judge the item yourself or rephrase the question. | | `jev-code: command not found` | `npx -y @french-castle/jev-code@latest doctor` works without a global install. | diff --git a/skills/jev/references/cli.md b/skills/jev/references/cli.md index 928bd55..5af4c85 100644 --- a/skills/jev/references/cli.md +++ b/skills/jev/references/cli.md @@ -2,7 +2,8 @@ When the `jev_*` tools are not registered in the current harness, the same operations are available from bash. The CLI reads the API key from the environment (`TYPESAFE_API_KEY`, -`OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`; the variable decides the host). +`OPENROUTER_API_KEY`, or `AI_GATEWAY_API_KEY`; the variable decides the host, and +`TYPESAFE_BASE_URL` points a `TYPESAFE_API_KEY` at a System One gateway of the user's own). ```bash jev-code doctor # is the key set, which harnesses are wired diff --git a/skills/jev/references/tools.md b/skills/jev/references/tools.md index 4cd3fb3..93d5aad 100644 --- a/skills/jev/references/tools.md +++ b/skills/jev/references/tools.md @@ -15,6 +15,10 @@ Shared conventions: TypeSafe, `typesafe/jev-1.13` through OpenRouter, `typesafe-ai/jev` through Vercel AI Gateway, `gpt-6-luna` on OpenAI's Decisions API, `nimble` (or the model you pulled) on a local Ollama. - Thresholds are optional inputs; the effective values are echoed under `thresholds`. +- A result whose question went unanswered carries `status` and the cautious outcome (`review`, + or `uncertain` for `jev_check`): `refused` when the host declined it, which OpenAI's Decisions + API can do for one question while answering the rest, and `invalid_response` when the answer + came back in a shape the tool cannot read. `jev_ask` lists declined ids under `refused`. ## jev_classify diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 417e76d..236fa8b 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -64,6 +64,8 @@ export async function runDoctor(options: DoctorOptions): Promise { ` ${"".padEnd(22)} ${variable.padEnd(20)} ${provider.label.padEnd(20)} ${provider.keysUrl}${optIn}`, ); } + // A malformed override is worth fixing before a key is even set. + if (config.problem) lines.push(` ${"".padEnd(22)} PROBLEM: ${config.problem}`); } else if (config.problem) { ok = false; lines.push(` ${"API key".padEnd(22)} PROBLEM: ${config.problem}`); @@ -142,7 +144,7 @@ export async function runDoctor(options: DoctorOptions): Promise { config.provider === "ollama" && error instanceof JevApiError && error.status === 404; lines.push( notEnabled - ? "Live check: FAILED: OpenAI's Decisions API is not enabled for this account; it is in limited preview. Use TypeSafe, OpenRouter, or Vercel AI Gateway meanwhile." + ? `Live check: FAILED: OpenAI refused this key access to the Decisions API (${errorMessage(error)}). It has been open to every developer since October 6, 2026, so check that the key's project may use ${config.model}.` : ollamaDown ? `Live check: FAILED: no Ollama server at ${config.baseUrl}. Start Ollama (0.35 or later), or point OLLAMA_HOST at it.` : ollamaNoModel diff --git a/src/cli/help.ts b/src/cli/help.ts index e8cfa29..edd6ec7 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -52,7 +52,7 @@ Environment TYPESAFE_API_KEY also follows TYPESAFE_BASE_URL and JEV_CODE_PROVIDER, as the TypeSafe SDK does. ${providerLines()} JEV_CODE_PROVIDER typesafe | openrouter | vercel | openai | ollama. Forces the host; required for openai and ollama. - TYPESAFE_BASE_URL Override the base URL (a proxy). Followed by TYPESAFE_API_KEY; other keys need JEV_CODE_PROVIDER. + TYPESAFE_BASE_URL A proxy, or a System One gateway of your own; /v1/systemone is appended. Followed by TYPESAFE_API_KEY; other keys need JEV_CODE_PROVIDER. TYPESAFE_DEFAULT_MODEL Override the host's default model (jev-latest; typesafe-ai/jev on Vercel). JEV_CODE_TIMEOUT_MS Default 30000 JEV_CODE_MAX_RETRIES Default 2 diff --git a/src/cli/run.ts b/src/cli/run.ts index a2a8398..27ab31b 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; import { homedir } from "node:os"; -import { JevClient } from "../core/client.js"; +import { JevClient, SYSTEM_ONE_PATH } from "../core/client.js"; import { describeConfig, describeProviderInUse, ENV } from "../core/config.js"; import { errorMessage, JevConfigError, JevValidationError } from "../core/errors.js"; import { @@ -220,7 +220,11 @@ async function setupCommand( } const KEY_PROMPT = - "No API key found. Paste a TypeSafe, OpenRouter, Vercel AI Gateway, or OpenAI key (input is hidden), or press Enter to skip: "; + "No API key found. Paste a TypeSafe, OpenRouter, Vercel AI Gateway, or OpenAI key, or one for a System One gateway of your own (input is hidden), or press Enter to skip: "; + +/** The last menu entry: a gateway the table does not know, reached through TYPESAFE_BASE_URL. */ +const OWN_GATEWAY = "gateway"; +const OWN_GATEWAY_LABEL = "Other System One gateway"; /** * Offer to take the key interactively when setup would otherwise register a tool that @@ -240,25 +244,64 @@ async function promptForKey( if (!key) return undefined; const named = providerByName(io.env[ENV.provider] ?? ""); const suggested = named && !named.keyless ? named : (providerForKey(key) ?? DEFAULT_PROVIDER); - const provider = await askHost(io, suggested); - // A pasted key is a decision, so an opt-in host gets its JEV_CODE_PROVIDER alongside. - const env: Record = { - [provider.keyEnv]: key, - ...(provider.explicitOnly ? { [ENV.provider]: provider.name } : {}), - }; + const host = await askHost(io, suggested); + const env = host === OWN_GATEWAY ? await gatewayEnv(io, key) : hostEnv(host, key); + if (!env) return undefined; const using = describeProviderInUse(describeConfig({ ...io.env, ...env })); if (using) io.stdout(`${using}\n\n`); return { env }; } +/** A pasted key is a decision, so an opt-in host gets its JEV_CODE_PROVIDER alongside. */ +function hostEnv(provider: Provider, key: string): Record { + return { + [provider.keyEnv]: key, + ...(provider.explicitOnly ? { [ENV.provider]: provider.name } : {}), + }; +} + /** - * Key shapes are not reliable, so the host is asked, with the best guess as the default. Only - * hosts that take a key are offered; a number, a name, or Enter answers. + * A gateway of your own speaks the System One API at an address the table does not know, so + * the key goes in TYPESAFE_API_KEY, the generic variable, with TYPESAFE_BASE_URL pointing at + * the gateway, exactly as the TypeSafe SDK is configured for a proxy. The result is judged like + * any other configuration, and nothing is stored when it is unusable (a malformed URL, say). */ -async function askHost(io: CliIO, suggested: Provider): Promise { +async function gatewayEnv(io: CliIO, key: string): Promise | undefined> { + if (!io.promptLine) return undefined; + const baseUrl = ( + await io.promptLine(`Base URL of the gateway (the client appends ${SYSTEM_ONE_PATH}): `) + ) + .trim() + .replace(/\/+$/, ""); + if (!baseUrl) { + io.stdout("No base URL given, so the key was not stored.\n\n"); + return undefined; + } + const model = ( + await io.promptLine(`Model id on the gateway, Enter for ${DEFAULT_PROVIDER.model}: `) + ).trim(); + const env = { + [ENV.apiKey]: key, + [ENV.baseUrl]: baseUrl, + ...(model ? { [ENV.model]: model } : {}), + }; + const { problem } = describeConfig({ ...io.env, ...env }); + if (problem) { + io.stdout(`${problem} The key was not stored.\n\n`); + return undefined; + } + return env; +} + +/** + * Key shapes are not reliable, so the host is asked, with the best guess as the default. Hosts + * that take a key are offered, plus a gateway of your own; a number, a name, or Enter answers. + */ +async function askHost(io: CliIO, suggested: Provider): Promise { const hosts = PROVIDERS.filter((provider) => !provider.keyless); if (!io.promptLine) return suggested; - const menu = hosts.map((provider, i) => `[${i + 1}] ${provider.label}`).join(" "); + const labels = [...hosts.map((provider) => provider.label), OWN_GATEWAY_LABEL]; + const menu = labels.map((label, i) => `[${i + 1}] ${label}`).join(" "); const answer = ( await io.promptLine( `Which host is this key for? ${menu}. Number or name, Enter for ${suggested.label}: `, @@ -267,6 +310,10 @@ async function askHost(io: CliIO, suggested: Provider): Promise { .trim() .toLowerCase(); if (!answer) return suggested; + const own = [String(labels.length), OWN_GATEWAY, "other", OWN_GATEWAY_LABEL.toLowerCase()]; + if (own.includes(answer)) { + return OWN_GATEWAY; + } const byNumber = hosts[Number(answer) - 1]; const byName = hosts.find((p) => p.name === answer || p.label.toLowerCase() === answer); const picked = byNumber ?? byName; @@ -275,8 +322,8 @@ async function askHost(io: CliIO, suggested: Provider): Promise { } function pastedKeyAdvice(env: Record): string { - const exports = Object.keys(env).map((name) => - name === ENV.provider ? ` export ${name}=${env[name]}` : ` export ${name}=...`, + const exports = Object.entries(env).map(([name, value]) => + /_KEY$/.test(name) ? ` export ${name}=...` : ` export ${name}=${value}`, ); return [ "", diff --git a/src/core/config.ts b/src/core/config.ts index 9554691..a829681 100644 --- a/src/core/config.ts +++ b/src/core/config.ts @@ -10,7 +10,7 @@ import { providerForKey, providerForUrl, } from "./providers.js"; -import type { WireName } from "./wires.js"; +import { SYSTEM_ONE_WIRE, type WireName } from "./wires.js"; /** * Environment variable names. The `TYPESAFE_*` names match the official SDKs; each host's @@ -78,6 +78,46 @@ function readPositiveInt(env: Env, name: string, fallback: number): number { return value; } +/** + * TYPESAFE_BASE_URL, checked before anything is sent. It names a proxy or a System One gateway + * of your own, and the client appends the endpoint path to it, so it must be an absolute + * http(s) URL with no credentials, query, or fragment, stopping before `/v1/systemone`. The + * messages never echo the value: it may embed a secret. + */ +function readBaseUrl(env: Env): string | undefined { + const raw = env[ENV.baseUrl]?.trim(); + if (!raw) return undefined; + const url = parseUrl(raw); + if (!url || (url.protocol !== "http:" && url.protocol !== "https:")) { + throw new JevConfigError(`${ENV.baseUrl} must be an absolute http(s) URL.`); + } + if (url.username || url.password) { + throw new JevConfigError( + `${ENV.baseUrl} must not embed credentials; the key goes in its own variable.`, + ); + } + if (/[?#]/.test(raw)) { + throw new JevConfigError( + `${ENV.baseUrl} must not carry a query or fragment; ${SYSTEM_ONE_WIRE.path} is appended to it.`, + ); + } + const base = raw.replace(/\/+$/, ""); + if (base.endsWith(SYSTEM_ONE_WIRE.path)) { + throw new JevConfigError( + `${ENV.baseUrl} must stop before ${SYSTEM_ONE_WIRE.path}; the client appends it.`, + ); + } + return base; +} + +function parseUrl(value: string): URL | undefined { + try { + return new URL(value); + } catch { + return undefined; + } +} + /** The provider JEV_CODE_PROVIDER names, when it names a known one. */ function explicitProvider(env: Env): Provider | undefined { const name = env[ENV.provider]?.trim(); @@ -138,14 +178,13 @@ export function serverUrlFromHost(value: string): string { } /** A keyless host named by JEV_CODE_PROVIDER needs no key; it still honours the overrides. */ -function selectKeyless(env: Env, provider: Provider): ProviderSelection { - const explicitUrl = env[ENV.baseUrl]?.trim(); +function selectKeyless( + env: Env, + provider: Provider, + explicitUrl: string | undefined, +): ProviderSelection { const hostValue = provider.hostEnv ? env[provider.hostEnv]?.trim() : undefined; - const baseUrl = explicitUrl - ? explicitUrl.replace(/\/+$/, "") - : hostValue - ? serverUrlFromHost(hostValue) - : provider.baseUrl; + const baseUrl = explicitUrl ?? (hostValue ? serverUrlFromHost(hostValue) : provider.baseUrl); const key = env[provider.keyEnv]?.trim() ?? ""; return { provider, @@ -281,7 +320,7 @@ export function selectProvider(env: Env): ProviderSelection { `${ENV.provider} must be one of ${PROVIDER_NAMES.join(", ")}, got "${explicitName}".`, ); } - const baseUrlOverride = env[ENV.baseUrl]?.trim() || undefined; + const baseUrlOverride = readBaseUrl(env); const urlProvider = baseUrlOverride ? providerForUrl(baseUrlOverride) : undefined; if (explicit && urlProvider && explicit.name !== urlProvider.name) { throw new JevConfigError( @@ -290,7 +329,7 @@ export function selectProvider(env: Env): ProviderSelection { } // A keyless host is reached only after the check above, so a stale base URL on another known // host cannot carry its key or the payload there; an unrecognised URL is a proxy it asked for. - if (explicit?.keyless) return selectKeyless(env, explicit); + if (explicit?.keyless) return selectKeyless(env, explicit, baseUrlOverride); const candidates = findKeys(env); const [first] = candidates; @@ -342,7 +381,7 @@ function finishSelection( provider, keyEnv: chosen.keyEnv, apiKey: chosen.key, - baseUrl: (baseUrlOverride ?? provider.baseUrl).replace(/\/+$/, ""), + baseUrl: baseUrlOverride ?? provider.baseUrl, model: env[ENV.model]?.trim() || provider.model, notes: selectionNotes(chosen, candidates, provider, baseUrlOverride, override), }; @@ -386,7 +425,7 @@ export interface ConfigSummary { maxRetries: number; /** Non-fatal observations worth showing, e.g. several keys set. */ notes: readonly string[]; - /** Why the configuration cannot be used, when keys are present but conflict. */ + /** Why the configuration cannot be used: keys that conflict, or a malformed override. */ problem?: string; } @@ -401,18 +440,29 @@ export function describeConfig(env: Env = process.env): ConfigSummary { return fallback; } }; + // A malformed base URL is one problem, reported here and not again by the selection. + const safeBaseUrl = (): string | undefined => { + try { + return readBaseUrl(env) ?? DEFAULTS.baseUrl; + } catch (error) { + problems.push(errorMessage(error)); + return undefined; + } + }; + const baseUrl = safeBaseUrl(); const [first] = findKeys(env); const keyless = explicitProvider(env)?.keyless === true; const base: ConfigSummary = { hasApiKey: first !== undefined || keyless, apiKeyHint: first ? maskSecret(first.key) : null, - baseUrl: (env[ENV.baseUrl]?.trim() || DEFAULTS.baseUrl).replace(/\/+$/, ""), + baseUrl: baseUrl ?? "", model: env[ENV.model]?.trim() || DEFAULTS.model, timeoutMs: safeInt(ENV.timeoutMs, DEFAULTS.timeoutMs), maxRetries: safeInt(ENV.maxRetries, DEFAULTS.maxRetries), notes: first ? [] : heldBackKeys(env).map(heldBackNote), }; - const selected = first || keyless ? selectOrReport(env, problems) : undefined; + const selected = + baseUrl !== undefined && (first || keyless) ? selectOrReport(env, problems) : undefined; const summary: ConfigSummary = selected ? { ...base, diff --git a/src/core/providers.ts b/src/core/providers.ts index 5c44d63..2714ccb 100644 --- a/src/core/providers.ts +++ b/src/core/providers.ts @@ -88,9 +88,10 @@ const VERCEL: Provider = Object.freeze({ }); /** - * OpenAI's Decisions API, in limited preview: the same three question types behind a different - * request shape (see wires.ts). Opt-in only, because OPENAI_API_KEY is set in many shells for - * other reasons and most accounts do not have Decisions access yet. + * OpenAI's Decisions API, in public beta for every developer since 2026-10-06: the same three + * question types behind a different request shape (see wires.ts). Opt-in only, because + * OPENAI_API_KEY is set in many shells for other reasons, and sending agent evidence to another + * company is a decision, not a side effect of an exported key. */ const OPENAI: Provider = Object.freeze({ name: "openai", diff --git a/src/core/types.ts b/src/core/types.ts index 1e1faca..ec8cf85 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -77,5 +77,10 @@ export interface SystemOneRequest { export interface SystemOneResponse { model: string; answers: Record; + /** + * Question ids the host declined to answer, absent when there were none. OpenAI's Decisions + * API can refuse one question and answer the rest; a refused id has no entry in `answers`. + */ + refused?: string[]; usage?: Usage; } diff --git a/src/core/wires.ts b/src/core/wires.ts index 981f049..dd8adc6 100644 --- a/src/core/wires.ts +++ b/src/core/wires.ts @@ -57,16 +57,17 @@ export const SYSTEM_ONE_WIRE: Wire = { }; /* - * OpenAI's `POST /v1/decisions` (limited preview). Shape taken from traffic recorded by a - * preview user and from OpenAI's own client in the Codex repository: + * OpenAI's `POST /v1/decisions`, in public beta since 2026-10-06. Shape from OpenAI's API + * reference (https://developers.openai.com/api/reference/resources/decisions/methods/create): * { model, input, questions: [{ type: "predicate" | "choice" | "score", name, instructions, - * choices?: [{ value, description? }], levels?: [{ label, description }] }] } + * choices?: [{ value, description? }], levels?: [{ label, description? }] }] } * { model, answers: [{ type, name, probability | choice + probabilities + confidence | - * score + probabilities + confidence }], usage: { input_tokens, output_tokens } } + * score + probabilities + confidence } | { type: "refusal", name }], usage } * Differences from System One that the mapping absorbs: questions are an array with names, * yes/no criteria have no field of their own and are folded into the instructions, score - * levels are labelled "0", "1", ... so probabilities map back to positions, and no legend - * comes back, so it is rebuilt from the request. + * levels are labelled "0", "1", ... and their probabilities come back keyed by level index, + * no legend comes back, so it is rebuilt from the request, and a question the model declines + * comes back as a refusal, listed under `refused`. */ interface DecisionsQuestion { type: "predicate" | "choice" | "score"; @@ -111,13 +112,20 @@ function toDecisionsQuestion(name: string, question: Question): DecisionsQuestio } } -function distribution(entries: unknown, key: "value" | "label"): Record { +/** + * Probabilities arrive as `[{ value, probability }]`; key them by the first field present. + * Choices are keyed by their value; score levels by their index (`value`), then their label. + */ +function distribution( + entries: unknown, + keys: readonly ("value" | "label")[], +): Record { const out: Record = {}; if (!Array.isArray(entries)) return out; for (const entry of entries) { if (!isRecord(entry) || typeof entry.probability !== "number") continue; - const id = entry[key] ?? entry.value; - if (id !== undefined && id !== null) out[String(id)] = entry.probability; + const id = keys.map((key) => entry[key]).find((value) => value !== undefined && value !== null); + if (id !== undefined) out[String(id)] = entry.probability; } return out; } @@ -134,7 +142,7 @@ function toAnswer(raw: Record, question: Question): Answer | un return { type: "choice", choice: raw.choice, - probabilities: distribution(raw.probabilities, "value"), + probabilities: distribution(raw.probabilities, ["value"]), confidence, }; case "score": { @@ -147,7 +155,7 @@ function toAnswer(raw: Record, question: Question): Answer | un type: "score", score: raw.score, legend, - probabilities: distribution(raw.probabilities, "label"), + probabilities: distribution(raw.probabilities, ["value", "label"]), confidence, }; } @@ -179,17 +187,25 @@ export const OPENAI_DECISIONS_WIRE: Wire = { throw new JevApiError("Decisions API returned a body without answers.", 200, requestId, body); } const answers: Record = {}; + const declined = new Set(); for (const raw of body.answers) { if (!isRecord(raw) || typeof raw.name !== "string") continue; const question = request.questions[raw.name]; if (!question) continue; + if (raw.type === "refusal") { + declined.add(raw.name); + continue; + } const answer = toAnswer(raw, question); if (answer) answers[raw.name] = answer; } + // A question both refused and answered counts as answered. + const refused = [...declined].filter((name) => !answers[name]); const usage = toUsage(body.usage); return { model: typeof body.model === "string" ? body.model : (request.model ?? model), answers, + ...(refused.length ? { refused } : {}), ...(usage ? { usage } : {}), }; }, diff --git a/src/tools/check.ts b/src/tools/check.ts index 3c90ca7..15af3af 100644 --- a/src/tools/check.ts +++ b/src/tools/check.ts @@ -1,7 +1,7 @@ import type { JevClient, RequestOptions } from "../core/client.js"; import { JevValidationError } from "../core/errors.js"; import type { NoulQuestion, Questions, State, Usage } from "../core/types.js"; -import { assertRequestBudget, round } from "./common.js"; +import { assertRequestBudget, type MissingStatus, missingStatus, round } from "./common.js"; import { parseInput, type ToolDefinition } from "./definition.js"; import { assertOrderedThresholds, checkVerdict, type Verdict } from "./policy.js"; import { type CheckInput, checkInputSchema } from "./schemas.js"; @@ -13,7 +13,7 @@ export interface CheckResult { question: string; probability: number; verdict: Verdict; - status?: "invalid_response"; + status?: MissingStatus; } export interface CheckOutput { @@ -67,7 +67,7 @@ export async function runCheck( question: wording[id] ?? "", probability: 0, verdict: "uncertain", - status: "invalid_response", + status: missingStatus(response, id), }; } const verdict = checkVerdict(answer.noul, yesAt, noAt); diff --git a/src/tools/classify.ts b/src/tools/classify.ts index 9036224..e37038e 100644 --- a/src/tools/classify.ts +++ b/src/tools/classify.ts @@ -6,6 +6,8 @@ import { buildState, contextValue, itemsToState, + type MissingStatus, + missingStatus, prepareItems, round, } from "./common.js"; @@ -24,7 +26,7 @@ export interface ClassifyResult { decision: Decision; probabilities: Record; truncated?: true; - status?: "invalid_response"; + status?: MissingStatus; } export interface ClassifyOutput { @@ -79,7 +81,7 @@ export async function runClassify( confidence: 0, decision: "review", probabilities: {}, - status: "invalid_response", + status: missingStatus(response, item.key), ...(item.truncated ? { truncated: true as const } : {}), }; } diff --git a/src/tools/common.ts b/src/tools/common.ts index fe46bb1..34a9e5c 100644 --- a/src/tools/common.ts +++ b/src/tools/common.ts @@ -1,6 +1,13 @@ import { JevValidationError } from "../core/errors.js"; import { LIMITS, TRUNCATION_MARKER } from "../core/limits.js"; -import type { JsonObject, JsonValue, State, SystemOneRequest, Text } from "../core/types.js"; +import type { + JsonObject, + JsonValue, + State, + SystemOneRequest, + SystemOneResponse, + Text, +} from "../core/types.js"; export interface Item { id: string; @@ -84,6 +91,16 @@ export function assertRequestBudget(request: SystemOneRequest): void { } } +/** + * Why a question has no usable answer: the host declined it (`refused`, which a retry will not + * change), or it answered in a shape the tool cannot read (`invalid_response`). + */ +export type MissingStatus = "refused" | "invalid_response"; + +export function missingStatus(response: SystemOneResponse, key: string): MissingStatus { + return response.refused?.includes(key) ? "refused" : "invalid_response"; +} + /** Round to 3 decimals so JSON output stays readable. */ export function round(value: number): number { return Math.round(value * 1000) / 1000; diff --git a/src/tools/rank.ts b/src/tools/rank.ts index 3b5e803..430b118 100644 --- a/src/tools/rank.ts +++ b/src/tools/rank.ts @@ -5,6 +5,8 @@ import { buildState, contextValue, itemsToState, + type MissingStatus, + missingStatus, prepareItems, round, } from "./common.js"; @@ -20,7 +22,7 @@ export interface RankResult { relevance: number; relevant: boolean; truncated?: true; - status?: "invalid_response"; + status?: MissingStatus; } export interface RankOutput { @@ -81,6 +83,7 @@ export async function runRank( id: candidate.id, relevance: valid ? answer.noul : 0, invalid: !valid, + status: missingStatus(response, candidate.key), truncated: candidate.truncated, }; }); @@ -95,7 +98,7 @@ export async function runRank( relevance: round(entry.relevance), relevant: !entry.invalid && entry.relevance >= relevantAt, ...(entry.truncated ? { truncated: true as const } : {}), - ...(entry.invalid ? { status: "invalid_response" as const } : {}), + ...(entry.invalid ? { status: entry.status } : {}), })); return { diff --git a/src/tools/score.ts b/src/tools/score.ts index 841686a..12a151d 100644 --- a/src/tools/score.ts +++ b/src/tools/score.ts @@ -6,6 +6,8 @@ import { buildState, contextValue, itemsToState, + type MissingStatus, + missingStatus, prepareItems, round, } from "./common.js"; @@ -27,7 +29,7 @@ export interface ScoreResult { decision: Decision; probabilities?: Record; truncated?: true; - status?: "invalid_response"; + status?: MissingStatus; } export interface ScoreOutput { @@ -87,7 +89,7 @@ export async function runScore( label: input.levels[0] ?? "", confidence: 0, decision: "review", - status: "invalid_response", + status: missingStatus(response, item.key), ...(item.truncated ? { truncated: true as const } : {}), }; } diff --git a/tests/cli/run.test.ts b/tests/cli/run.test.ts index f9a6943..e4d364f 100644 --- a/tests/cli/run.test.ts +++ b/tests/cli/run.test.ts @@ -291,7 +291,7 @@ describe("runCli", () => { expect(asked).toHaveLength(2); expect(asked[0]).toMatch(/TypeSafe, OpenRouter, Vercel AI Gateway, or OpenAI key/); expect(asked[1]).toMatch( - /Which host is this key for\? \[1\] TypeSafe {2}\[2\] OpenRouter {2}\[3\] Vercel AI Gateway {2}\[4\] OpenAI Decisions API\. Number or name, Enter for OpenRouter: /, + /Which host is this key for\? \[1\] TypeSafe {2}\[2\] OpenRouter {2}\[3\] Vercel AI Gateway {2}\[4\] OpenAI Decisions API {2}\[5\] Other System One gateway\. Number or name, Enter for OpenRouter: /, ); const config = JSON.parse( readFileSync(join(s.home, ".config", "opencode", "opencode.json"), "utf8"), @@ -347,7 +347,7 @@ describe("runCli", () => { expect(s.out()).not.toContain("abcdefghijkl"); }); - it("explains the OpenAI opt-in in doctor and recognises the preview's 403", async () => { + it("explains the OpenAI opt-in in doctor and explains a 403 from the Decisions API", async () => { const ambient = io({ env: { OPENAI_API_KEY: "sk-proj-abcdefghijkl" } }); expect(await runCli(["doctor"], ambient.io)).toBe(1); expect(ambient.out()).toContain("NOT SET"); @@ -371,7 +371,10 @@ describe("runCli", () => { expect(await runCli(["doctor", "--live"], preview.io)).toBe(1); expect(preview.out()).toMatch(/provider\s+OpenAI Decisions API \(OPENAI_API_KEY/); expect(preview.out()).toContain("https://api.openai.com"); - expect(preview.out()).toContain("not enabled for this account"); + expect(preview.out()).toContain("OpenAI refused this key access to the Decisions API"); + expect(preview.out()).toContain("Decision API is not enabled for this user."); + expect(preview.out()).toContain("check that the key's project may use gpt-6-luna"); + expect(preview.out()).not.toMatch(/limited preview/); }); it("runs Ollama without a key and explains a missing server or model", async () => { @@ -451,4 +454,100 @@ describe("runCli", () => { expect(conflicting.out()).toContain("no key is set for Vercel AI Gateway"); expect(conflicting.out()).toContain("No key was written into any config"); }); + + it("stores a pasted gateway key in the SDK's variables, with the gateway's URL and model", async () => { + const asked: string[] = []; + const answers = ["5", " https://gateway.example/api/ ", " vendor/jev "]; + const s = io({ + env: {}, + promptSecret: async () => "gw-opaque-key-1234", + promptLine: async (question) => { + asked.push(question); + return answers.shift() ?? ""; + }, + }); + expect(await runCli(["setup", "opencode"], s.io)).toBe(0); + expect(asked[0]).toContain("[5] Other System One gateway"); + expect(asked[1]).toContain("the client appends /v1/systemone"); + expect(asked[2]).toContain("Enter for jev-latest"); + const stored = (home: string) => + JSON.parse(readFileSync(join(home, ".config", "opencode", "opencode.json"), "utf8")).mcp.jev + .environment; + expect(stored(s.home)).toEqual({ + TYPESAFE_API_KEY: "gw-opaque-key-1234", + TYPESAFE_BASE_URL: "https://gateway.example/api", + TYPESAFE_DEFAULT_MODEL: "vendor/jev", + }); + expect(s.out()).toContain("Using TypeSafe (TYPESAFE_API_KEY gw-o…1234)"); + expect(s.out()).toContain("Requests go to https://gateway.example/api (TYPESAFE_BASE_URL)"); + expect(s.out()).toContain("export TYPESAFE_API_KEY=..."); + expect(s.out()).toContain("export TYPESAFE_BASE_URL=https://gateway.example/api"); + expect(s.out()).toContain("export TYPESAFE_DEFAULT_MODEL=vendor/jev"); + expect(s.out()).not.toContain("opaque"); + // By name, keeping the default model: no TYPESAFE_DEFAULT_MODEL is stored. + const script = (lines: string[]) => async () => lines.shift() ?? ""; + const byName = io({ + env: {}, + promptSecret: async () => "gw-opaque-key-1234", + promptLine: script(["gateway", "https://gateway.example", ""]), + }); + expect(await runCli(["setup", "opencode"], byName.io)).toBe(0); + expect(stored(byName.home)).toEqual({ + TYPESAFE_API_KEY: "gw-opaque-key-1234", + TYPESAFE_BASE_URL: "https://gateway.example", + }); + // Without a URL the key has no destination, so it is not stored. + const noUrl = io({ + env: {}, + promptSecret: async () => "gw-opaque-key-1234", + promptLine: script(["5", ""]), + }); + expect(await runCli(["setup", "opencode"], noUrl.io)).toBe(0); + expect(noUrl.out()).toContain("No base URL given, so the key was not stored."); + expect(noUrl.out()).toContain("No API key found in this shell"); + expect(stored(noUrl.home)).toBeUndefined(); + // The full endpoint pasted as the base URL is refused, and nothing is stored. + const endpoint = io({ + env: {}, + promptSecret: async () => "gw-opaque-key-1234", + promptLine: script(["5", "https://gateway.example/v1/systemone", ""]), + }); + expect(await runCli(["setup", "opencode"], endpoint.io)).toBe(0); + expect(endpoint.out()).toContain( + "TYPESAFE_BASE_URL must stop before /v1/systemone; the client appends it. The key was not stored.", + ); + expect(endpoint.out()).not.toContain("Using "); + expect(endpoint.out()).not.toContain("export TYPESAFE_BASE_URL"); + expect(stored(endpoint.home)).toBeUndefined(); + }); + + it("shows a gateway in doctor and reports a malformed base URL without echoing it", async () => { + const gateway = io({ + env: { + TYPESAFE_API_KEY: "gw-opaque-key-1234", + TYPESAFE_BASE_URL: "https://gateway.example/api", + TYPESAFE_DEFAULT_MODEL: "vendor/jev", + }, + }); + expect(await runCli(["doctor"], gateway.io)).toBe(0); + expect(gateway.out()).toMatch(/provider\s+TypeSafe \(TYPESAFE_API_KEY gw-o…1234\)/); + expect(gateway.out()).toMatch(/base URL\s+https:\/\/gateway\.example\/api/); + expect(gateway.out()).toMatch(/model\s+vendor\/jev/); + expect(gateway.out()).toContain("Requests go to https://gateway.example/api"); + const leaky = io({ + env: { + TYPESAFE_API_KEY: "gw-opaque-key-1234", + TYPESAFE_BASE_URL: "https://user:secret@gateway.example", + }, + }); + expect(await runCli(["doctor", "--live"], leaky.io)).toBe(1); + expect(leaky.out()).toContain("PROBLEM: TYPESAFE_BASE_URL must not embed credentials"); + expect(leaky.out()).toContain("Live check: skipped, fix the configuration problem"); + expect(leaky.out()).not.toContain("secret"); + // Without a key, the URL problem still shows under the list of keys to set. + const keyless = io({ env: { TYPESAFE_BASE_URL: "gateway.example" } }); + expect(await runCli(["doctor"], keyless.io)).toBe(1); + expect(keyless.out()).toContain("NOT SET"); + expect(keyless.out()).toContain("PROBLEM: TYPESAFE_BASE_URL must be an absolute http(s) URL."); + }); }); diff --git a/tests/core/config.test.ts b/tests/core/config.test.ts index 185381e..2ef64e1 100644 --- a/tests/core/config.test.ts +++ b/tests/core/config.test.ts @@ -117,6 +117,48 @@ describe("resolveConfig", () => { }); }); + it("takes a System One gateway of your own through TYPESAFE_BASE_URL, checking the URL first", () => { + const gateway = resolveConfig({ + TYPESAFE_API_KEY: "gw-key", + TYPESAFE_BASE_URL: "https://gateway.example/api/", + TYPESAFE_DEFAULT_MODEL: "vendor/jev", + }); + expect(gateway).toMatchObject({ + provider: "typesafe", + wire: "systemone", + keyEnv: "TYPESAFE_API_KEY", + baseUrl: "https://gateway.example/api", + model: "vendor/jev", + }); + const malformed: Array<[string, RegExp]> = [ + ["gateway.example", /absolute http\(s\) URL/], + ["ftp://gateway.example", /absolute http\(s\) URL/], + ["https://user:secret@gateway.example", /must not embed credentials/], + [ + "https://gateway.example/api?token=secret", + /query or fragment; \/v1\/systemone is appended/, + ], + ["https://gateway.example/api#secret", /query or fragment/], + ["https://gateway.example/v1/systemone/", /must stop before \/v1\/systemone/], + ]; + for (const [url, reason] of malformed) { + const env = { TYPESAFE_API_KEY: "gw-key", TYPESAFE_BASE_URL: url }; + expect(() => resolveConfig(env)).toThrow(reason); + expect(() => resolveConfig(env)).toThrow(/^TYPESAFE_BASE_URL/); + // Reported once, as a problem, with the value kept out of every field. + const summary = describeConfig(env); + expect(summary.problem).toMatch(reason); + expect(summary.problem?.match(/TYPESAFE_BASE_URL/g)).toHaveLength(1); + expect(summary).toMatchObject({ hasApiKey: true, baseUrl: "", notes: [] }); + expect(summary.provider).toBeUndefined(); + expect(JSON.stringify(summary)).not.toContain("secret"); + } + // A keyless host's override is checked the same way. + expect(() => + resolveConfig({ JEV_CODE_PROVIDER: "ollama", TYPESAFE_BASE_URL: "localhost:11434" }), + ).toThrow(/^TYPESAFE_BASE_URL must be an absolute/); + }); + it("lets TYPESAFE_API_KEY follow JEV_CODE_PROVIDER and TYPESAFE_BASE_URL like the TypeSafe SDK", () => { // The SDK's own OpenRouter and Vercel guides set TYPESAFE_API_KEY plus the base URL. const viaUrl = resolveConfig({ diff --git a/tests/core/wires.test.ts b/tests/core/wires.test.ts index 8d1915e..c7ceca3 100644 --- a/tests/core/wires.test.ts +++ b/tests/core/wires.test.ts @@ -3,7 +3,10 @@ import { JevApiError } from "../../src/core/errors.js"; import type { SystemOneRequest } from "../../src/core/types.js"; import { OPENAI_DECISIONS_WIRE, SYSTEM_ONE_WIRE, WIRES } from "../../src/core/wires.js"; -/** The three-question request a preview user recorded against api.openai.com/v1/decisions. */ +/** + * Three questions, one of each type. The answer body below was recorded against + * api.openai.com/v1/decisions and matches OpenAI's published reference. + */ const request: SystemOneRequest = { state: { message: "I was charged twice." }, questions: { @@ -189,31 +192,64 @@ describe("OpenAI Decisions wire", () => { }); }); - it("maps score probabilities by label, falling back to value, and ignores junk entries", () => { + it("keys score probabilities by level index, falling back to the label, and ignores junk entries", () => { const body = { answers: [ { type: "score", name: "frustration", - score: 2, + score: 1.1, probabilities: [ - { value: 0, probability: 0.1 }, - { value: 2, probability: 0.9 }, + // The reference documents `value` as the level index; the label is only echoed. + { value: 0, label: "Calm", probability: 0.1 }, + { value: 2, label: "Angry", probability: 0.2 }, + { label: "1", probability: 0.7 }, "junk", { value: 1 }, ], + confidence: 0.55, }, ], }; const response = OPENAI_DECISIONS_WIRE.decode(body, request, "m"); expect(response.answers.frustration).toMatchObject({ type: "score", - score: 2, - probabilities: { "0": 0.1, "2": 0.9 }, - confidence: 0, + score: 1.1, + probabilities: { "0": 0.1, "1": 0.7, "2": 0.2 }, + confidence: 0.55, }); }); + it("lists the questions the model declines under refused, and keeps the others' answers", () => { + const body = { + model: "gpt-6-luna", + answers: [ + { type: "refusal", name: "urgent" }, + { type: "choice", name: "department", choice: "billing" }, + { type: "refusal", name: "frustration" }, + // Unknown and unnamed refusals are not ours to report. + { type: "refusal", name: "unknown" }, + { type: "refusal", name: null }, + ], + }; + const response = OPENAI_DECISIONS_WIRE.decode(body, request, "m"); + expect(Object.keys(response.answers)).toEqual(["department"]); + expect(response.refused).toEqual(["urgent", "frustration"]); + // A refusal followed by an answer for the same question counts as answered. + const both = OPENAI_DECISIONS_WIRE.decode( + { + answers: [ + { type: "refusal", name: "urgent" }, + { type: "predicate", name: "urgent", probability: 0.4 }, + ], + }, + request, + "m", + ); + expect(both.answers.urgent).toEqual({ type: "noul", noul: 0.4 }); + expect(both.refused).toBeUndefined(); + }); + it("drops a known question's answer when its payload is malformed, lets the last duplicate win, and ignores odd usage", () => { const body = { model: 7, diff --git a/tests/e2e/live.test.ts b/tests/e2e/live.test.ts index a460874..6fb7ac2 100644 --- a/tests/e2e/live.test.ts +++ b/tests/e2e/live.test.ts @@ -1,7 +1,9 @@ /** - * Live tests against a real Jev host. Run with a TypeSafe, OpenRouter, or Vercel AI Gateway key: + * Live tests against a real host. Run with a TypeSafe, OpenRouter, or Vercel AI Gateway key, or + * against OpenAI's Decisions API: * TYPESAFE_API_KEY=... npm run test:e2e * OPENROUTER_API_KEY=... npm run test:e2e + * OPENAI_API_KEY=... JEV_CODE_PROVIDER=openai npm run test:e2e * They cost a few hundred tokens and are skipped without a usable key. */ import { describe, expect, it } from "vitest"; diff --git a/tests/setup/configs.test.ts b/tests/setup/configs.test.ts index 3fa39e2..a56f555 100644 --- a/tests/setup/configs.test.ts +++ b/tests/setup/configs.test.ts @@ -46,10 +46,10 @@ describe("config editors", () => { }); expect( serverEnvFromProcess( - { TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "u", OTHER: "x" }, + { TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "https://u.test", OTHER: "x" }, { includeApiKey: true }, ), - ).toEqual({ TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "u" }); + ).toEqual({ TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "https://u.test" }); expect(serverEnvFromProcess({ TYPESAFE_API_KEY: "k" }, { includeApiKey: false })).toEqual({}); expect(() => toSpec([], {})).toThrow(/must not be empty/); }); diff --git a/tests/tools/registry.test.ts b/tests/tools/registry.test.ts index 825a0d8..a93ccc7 100644 --- a/tests/tools/registry.test.ts +++ b/tests/tools/registry.test.ts @@ -1,5 +1,30 @@ import { describe, expect, it } from "vitest"; +import { JevClient } from "../../src/core/client.js"; +import { OPENAI_DECISIONS_WIRE } from "../../src/core/wires.js"; import { findTool, TOOL_NAMES, TOOLS, USAGE_GUIDANCE } from "../../src/tools/index.js"; +import { fakeFetch, jsonResponse } from "../helpers.js"; + +/** A client on OpenAI's wire whose host declines every question it is asked. */ +function refusingClient() { + const { fetch, calls } = fakeFetch([ + (body) => { + const sent = body as unknown as { questions: Array<{ name: string }> }; + return jsonResponse({ + model: "gpt-6-luna", + answers: sent.questions.map(({ name }) => ({ type: "refusal", name })), + }); + }, + ]); + const client = new JevClient({ + apiKey: "sk-proj-test", + baseUrl: "https://api.openai.com", + model: "gpt-6-luna", + wire: OPENAI_DECISIONS_WIRE, + fetch, + maxRetries: 0, + }); + return { client, calls }; +} describe("tool registry", () => { it("exposes five uniquely named tools with complete metadata", () => { @@ -20,3 +45,36 @@ describe("tool registry", () => { expect(findTool("nope")).toBeUndefined(); }); }); + +describe("a question the host declines", () => { + const items = [{ id: "a/1", text: "Export fails in Safari." }]; + + it("is reported as refused, never as a guess, by every tool", async () => { + const run = async (name: string, payload: unknown) => { + const { client, calls } = refusingClient(); + const output = await findTool(name)?.run(client, payload); + expect(calls[0]?.url).toBe("https://api.openai.com/v1/decisions"); + return output as Record; + }; + const check = await run("check", { state: "s", checks: { ok: "Is it fine?" } }); + expect(check.results[0]).toMatchObject({ verdict: "uncertain", status: "refused" }); + expect(check.summary).toMatchObject({ uncertain: 1, yes: 0, no: 0 }); + const classify = await run("classify", { items, classes: { bug: null, other: null } }); + expect(classify.results[0]).toMatchObject({ + id: "a/1", + label: null, + decision: "review", + status: "refused", + }); + const score = await run("score", { items, instructions: "Severity?", levels: ["low", "high"] }); + expect(score.results[0]).toMatchObject({ decision: "review", status: "refused" }); + const rank = await run("rank", { query: "Safari", candidates: items }); + expect(rank.ranked[0]).toMatchObject({ relevant: false, status: "refused" }); + expect(rank.any_relevant).toBe(0); + const ask = await run("ask", { + state: "s", + questions: { q: { type: "noul", instructions: "Is it fine?" } }, + }); + expect(ask).toMatchObject({ answers: {}, refused: ["q"] }); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index ec1e712..e50d2db 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,7 +3,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { include: ["tests/**/*.test.ts"], - exclude: ["tests/e2e/**"], + // The live tests cost tokens, so they run only through `npm run test:e2e`, which sets + // JEV_CODE_E2E=1; excluding them unconditionally left that script with no files to run. + exclude: process.env.JEV_CODE_E2E === "1" ? [] : ["tests/e2e/**"], environment: "node", coverage: { provider: "v8",