diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..b36811f --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,15 @@ +{ + "name": "modelport", + "description": "ModelPort: upgrade LLM code to newer models without breaking prod", + "owner": { + "name": "Kumar Adarsh Pandita", + "url": "https://github.com/forkadarshp" + }, + "plugins": [ + { + "name": "modelport", + "source": "./", + "description": "Migrate prompts, tools, API calls, and tests to a newer LLM, with per-model breaking-change guides and proof that the upgrade worked." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..3aa6366 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,14 @@ +{ + "name": "modelport", + "displayName": "ModelPort", + "version": "0.2.0", + "description": "Upgrade LLM code to newer models (Claude 5.x, GPT-6, Gemini 3.8) without breaking prod. Applies each model's breaking-change fixes, then proves the upgrade with tests and evals.", + "author": { + "name": "Kumar Adarsh Pandita", + "url": "https://github.com/forkadarshp" + }, + "homepage": "https://github.com/forkadarshp/MPort", + "repository": "https://github.com/forkadarshp/MPort", + "license": "MIT", + "keywords": ["model-migration", "llm-migration", "prompt-migration", "model-upgrade", "breaking-changes", "evals"] +} diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml new file mode 100644 index 0000000..7df2730 --- /dev/null +++ b/.github/workflows/freshness.yml @@ -0,0 +1,38 @@ +name: Model guide freshness + +on: + schedule: + - cron: "0 9 * * 1" # Mondays 09:00 UTC + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check guide ages + id: check + continue-on-error: true + run: python3 scripts/check_freshness.py . | tee freshness-report.txt + - name: Open or update the freshness issue + if: steps.check.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + run: | + title="Model guides need re-verification" + body="$(cat freshness-report.txt)" + existing="$(gh issue list --state open --search "$title in:title" --json number --jq '.[0].number')" + if [ -n "$existing" ]; then + gh issue comment "$existing" --body "$body" + else + gh issue create --title "$title" --body "$body" + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5695dd8..fb670f3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,6 +34,23 @@ python3 scripts/validate_skill.py . npx markdownlint-cli2 "**/*.md" ``` +## Adding or updating a model guide + +New models are the most valuable contribution. A guide needs: + +- A header with `Model ID:`, pricing, `Last verified: YYYY-MM-DD`, and + `Source:` (the provider page you used). Mark facts from blogs or other + secondary sources as such. +- Required changes (will error), silent changes (behavior differs), upgrade + paths from older models, and optional tuning, kept separate. +- An entry in `references/models/INDEX.md`, and in + `references/breaking-changes.md` if the model adds new errors. +- Superseded guides get `Status: legacy` and `Last verified: legacy, ...` + instead of being deleted. + +`python3 scripts/validate_skill.py .` checks the header and the index entry. +`python3 scripts/check_freshness.py .` lists guides older than 30 days. + ## Reporting bugs Use the bug template and include: diff --git a/README.md b/README.md index 58bc13a..80f6e79 100644 --- a/README.md +++ b/README.md @@ -2,63 +2,86 @@ [![CI](https://github.com/forkadarshp/MPort/actions/workflows/ci.yml/badge.svg)](https://github.com/forkadarshp/MPort/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -[![Codex Skill](https://img.shields.io/badge/Codex-Skill-111827)](SKILL.md) +[![Agent Skill](https://img.shields.io/badge/Agent-Skill-111827)](SKILL.md) +[![Models](https://img.shields.io/badge/Claude_5.x_·_GPT--6_·_Gemini_3.8-guides-blue)](references/models/INDEX.md) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md) -[![Status: Alpha](https://img.shields.io/badge/Status-v0.1_Alpha-blue)](docs/PUBLISHING.md) ![ModelPort social preview](assets/social-preview.svg) -**Ship model upgrades without breaking prod — then prove it.** +**Upgrade to Claude Opus 5.5, GPT-6, or Gemini 3.8 without breaking prod.** -Two things in one agent skill: **behavior-preserving migrations** across prompts, agents, tools, API callers, and tests — and **built-in evals** that score your old vs. new system so every upgrade is backed by numbers, not vibes. Works in Claude Code, Codex, and Cursor. +ModelPort is an agent skill for Claude Code, Codex, and Cursor. It migrates the +prompts, tool definitions, API calls, and tests around a model swap, using +per-model guides that track each provider's breaking changes. Then it proves the +upgrade worked with tests, a live smoke check, and optional old-vs-new evals. -> **Install in one line:** `npx skills add forkadarshp/MPort` — then tell your agent to migrate. See [Quickstart](#quickstart). +> **Install:** `npx skills add forkadarshp/MPort`, then tell your agent +> *"migrate this repo to claude-opus-5-5"*. See [Quickstart](#quickstart). -**The model ID is one line. The behavior around it is everything else.** -Most migrations break because search-and-replace silently shatters tool calling, -parsers, and orchestration. ModelPort treats a swap as behavior preservation: +## Seen one of these after a model swap? ```text - # the naive migration ┄ what a find/replace does - $ grep -rl 'opus-4-7' . | xargs sed -i 's/4-7/4-8/g' - x tool calls break x parser drift x silent prod regressions - - # the ModelPort migration ┄ behavior-preserving - > migrate this repo to claude-opus-4-8, keep behavior, prove it - ✓ callers mapped ✓ contract held ✓ tests + rollback evidence +"thinking.type.disabled" is not supported for this model. +tool_choice: type "tool" and "any" are not supported for this model. +'claude-opus-5-5' does not support tool types: computer_20251124. +temperature / top_p / logprobs rejected after moving to GPT-6 +thinking_budget and thinking_level in the same request (Gemini 3.x) → 400 +stop_reason: "refusal" and code that reads content[0] ``` -> **Star it** if it's saved you a migration headache — it helps other teams find it. +Changing the model ID is one line. The 400s, the truncated replies (thinking now +counts toward `max_tokens`), the quiet UI between tool calls, and the parsers +that stop matching are everything else. The lookup for all of them is in +[`references/breaking-changes.md`](references/breaking-changes.md). ModelPort +applies the fixes for you, in order, and shows its evidence. + +## What you get + +- **Current-model knowledge.** Dedicated guides for Claude Fable 5.1, Opus 5.5, + Sonnet 5.5, Haiku 5.5, GPT-6, and Gemini 3.8 Flash. Older guides layer + underneath for upgrades that start further back. +- **Guides that stay fresh.** Every guide has a `Last verified:` date and a + `Source:`. A weekly job flags stale guides, and the skill re-checks the + provider's docs before trusting an old one. +- **A migration, not a find/replace.** Scope lock, caller discovery, file + classification, required fixes kept apart from optional tuning, then + validation and rollback notes. +- **Proof.** Tests, a live smoke call, and an optional three-arm benchmark + (old model, naive swap, ModelPort) with a runnable harness. + +## Supported models + +| Provider | Models with a dedicated guide | +| :------- | :---------------------------- | +| Anthropic | Claude Fable 5.1, Opus 5.5, Sonnet 5.5, Haiku 5.5 (legacy: Opus 4.8, Opus 4.6, Sonnet 4.5) | +| OpenAI | GPT-6 Astra, GPT-6.1 Sol, GPT-6 Sol, GPT-6 Luna (legacy: GPT-5.5) | +| Google | Gemini 3.8 Flash | +| Anything else | Generic playbook plus provider checklists, with research on the target's docs | -## Now shipping: Claude Opus 4.8 (released 2026-05-28) +Registry, upgrade paths, and freshness rules: [`references/models/INDEX.md`](references/models/INDEX.md). -Opus 4.8 is live, and ModelPort already ships a dedicated migration guide for it: -[`references/models/claude-opus-4-8.md`](references/models/claude-opus-4-8.md). -It covers the **4.7 → 4.8 drop-in path**, the **4.6 → 4.7 breaking changes you -must apply first** (non-default sampling params now 400, manual extended -thinking → `{type: "adaptive"}` + `effort`, new tokenizer, removed assistant -prefills), and **effort re-baselining**. Point the skill at your repo: +## Example ```text -Use $modelport-skill to migrate this codebase to claude-opus-4-8. +Use $modelport-skill to migrate this codebase to claude-opus-5-5. Apply required compatibility fixes first, keep tool + output behavior stable, and return validation evidence plus rollback notes. ``` +You get back a report with: what changed and why, required fixes versus +optional tuning, test and smoke results, risks, and how to roll back. A sample +is in [examples/example-output.md](examples/example-output.md). + ## How it works -ModelPort treats a model swap as a **behavior-preservation problem**, not a -search-and-replace. The model ID is one line; the behavior around it — tool -calls, parsers, output contracts, prompts, orchestration — is everything else. -The skill drives a strict, phase-gated pipeline that maps every caller, -classifies each hit before editing, keeps required fixes apart from optional -tuning, and refuses to claim "done" without proof. +A strict, phase-gated pipeline. The skill will not edit before scope is locked +and will not claim "done" without proof. ```text ╭──────────────────────────────────────────────────────────────────────╮ │ │ -│ [ Opus 4.7 ] ═══> ( ModelPort ) ═══> [ Opus 4.8 ] │ -│ drop-in upgrade behavior-preserving GA · 2026-05-28 │ +│ [ old model ] ═══> ( ModelPort ) ═══> [ new model ] │ +│ what you run behavior-preserving what you ship │ │ │ │ » phase 0 lock scope, read current state ················· [ OK ]│ │ » phase 1 discover callers, prompts, agents, tools ······· [ OK ]│ @@ -71,109 +94,101 @@ tuning, and refuses to claim "done" without proof. ╰──────────────────────────────────────────────────────────────────────╯ ``` -## More than a model-ID swap - -ModelPort doesn't just rewrite `model="..."`. It updates prompts, tool -descriptions, and guardrails to match how the **target model actually behaves** — -including the undocumented quirks that never make the changelog. - -Take chess: GPT-3.5 played badly, GPT-4o was a leap ahead — not from a rules -change, but because 4o's training data held far more chess games. That kind of -capability shift never appears in the release notes. ModelPort's research -surfaces these advertised *and* unadvertised behaviors — from observed runs, -prior migrations, web research, and provider changelogs — and folds the matching -prompt and tooling changes into the migration, so the new model is actually used -to its strengths instead of just dropped in. - -## Outcomes: with vs. without ModelPort - -Skipping the discipline is exactly where migrations look fine in review and -then silently regress in production — broken tool calls, drifted output -contracts, no proof, and no way back. +## Beyond the model ID -```text -╭───────────────────────────────────┬───────────────────────────────────╮ -│ WITHOUT ModelPort │ WITH ModelPort │ -├───────────────────────────────────┼───────────────────────────────────┤ -│ x find/replace the model ID │ ✓ scope-locked discovery │ -│ x tool calls break silently │ ✓ callers mapped, not guessed │ -│ x parser / output drift │ ✓ output contract preserved │ -│ x prompt hacks carry over │ ✓ fixes vs tuning kept apart │ -│ x "looks fine" — no proof │ ✓ tests + live smoke evidence │ -│ x no way back on breakage │ ✓ rollback notes included │ -├───────────────────────────────────┼───────────────────────────────────┤ -│ → silent prod regressions │ → behavior-preserving upgrade │ -╰───────────────────────────────────┴───────────────────────────────────╯ -``` +ModelPort also adjusts prompts, tool descriptions, and guardrails to match how +the target model behaves, including changes the changelog barely mentions. Two +examples from the current generation: Claude Opus 5.5 defaults to `medium` +effort, one level below Opus 5, so an untouched config quietly thinks less. And +Claude Sonnet 5.5 can read a user message that arrives right after a tool result +as a prompt injection, so mid-task user input must be delivered as a user turn. +Each guide lists these as required fixes or optional tuning, never mixed. -## Benchmark the upgrade — built-in evals +## Benchmark the upgrade -Migrations shouldn't be a leap of faith. Opt in and ModelPort ends with -**measured evidence, not vibes** — it runs the same eval set against three -configurations so the raw model delta and the skill's added value are attributed -separately: +Opt in and ModelPort ends with measured evidence. It runs the same eval set +against three configurations so the raw model delta and the skill's value are +attributed separately: -- **Baseline** — old model + old prompts (where you started) -- **Naive swap** — new model + old prompts (what a find/replace would get you) -- **ModelPort-enhanced** — new model + skill-enhanced system (where you land) +- **Baseline**: old model, old prompts (where you started) +- **Naive swap**: new model, old prompts (what a find/replace gets you) +- **ModelPort-enhanced**: new model, enhanced system (where you land) -A raw swap tends to regress exactly the things that don't survive a model change -on their own — output contracts, tool calls, and parse-able format — because -newer models follow instructions more literally and re-tokenize differently. -The enhanced arm is where the skill's fixes earn their keep. Illustrative -leaderboard (your run produces the actual measured numbers): +Sample leaderboard from the bundled **offline simulator**. The simulator models +only prompt explicitness and a per-model "literalness" knob, so these numbers +show the report format, not real model performance: ```text ╭──────────────────────┬────────────┬────────────┬────────────╮ │ metric │ baseline │ naive swap │ ModelPort │ │ │ (old/old) │ (new/old) │ (new/enh.) │ ├──────────────────────┼────────────┼────────────┼────────────┤ -│ task success │ 82% │ 84% │ 91% │ -│ output contract │ 93% │ 89% │ 98% │ -│ tool-call accuracy │ 90% │ 83% │ 96% │ -│ p95 latency │ 3.9s │ 2.4s │ 2.6s │ -│ cost / req │ $0.052 │ $0.047 │ $0.045 │ -│ refusal / halluc. │ 2.8% │ 2.1% │ 1.4% │ +│ task success │ 64% │ 57% │ 93% │ +│ output contract │ 64% │ 57% │ 93% │ +│ tool-call accuracy │ 64% │ 57% │ 93% │ +│ p95 latency │ 4.1s │ 2.5s │ 2.5s │ +│ cost / req │ $0.0008 │ $0.0006 │ $0.0009 │ ├──────────────────────┼────────────┼────────────┼────────────┤ -│ composite (50/30/20)│ 0.80 │ 0.83 │ 0.91 │ +│ composite (50/30/20)│ 0.57 │ 0.63 │ 0.76 │ ╰──────────────────────┴────────────┴────────────┴────────────╯ ``` -Numbers above are illustrative, not measured results — latency and cost in -particular move with the target model and are always reported as measured, never -assumed. Methodology, metric definitions, and composite scoring live in +Get real numbers: `python3 harness/run.py --provider anthropic` (needs +`ANTHROPIC_API_KEY`). The harness works with current Claude models: it never +sends parameters they reject and leaves room for always-on thinking. Details in +[harness/README.md](harness/README.md) and [references/benchmarking.md](references/benchmarking.md). -**Run it yourself.** A bundled harness turns this into a closed loop: -`python3 harness/run.py` scores the three arms on an eval set, and -`harness/iterate.py` sweeps prompt revisions so you watch the score climb until -it plateaus. Two scenarios ship (support triage + multi-tool routing); it runs -offline by default, or `--provider anthropic` for measured numbers. See -[harness/README.md](harness/README.md). - -## Why ModelPort - -- Replace deprecated model IDs without breaking runtime calls. -- Preserve output contracts, parser behavior, and tool calling. -- Separate required compatibility fixes from optional prompt tuning. -- Choose direct swap, adapter, shadow mode, canary, or phased rollout. -- Produce validation evidence and rollback notes. -- Benchmark pre vs. post (and raw-swap control) to quantify the upgrade. - ## Quickstart -Install once, use everywhere — the universal skills CLI wires ModelPort into Codex, Claude Code, Cursor, and more: +**1. Install.** Pick the line for your agent. ```bash +# Any agent (Claude Code, Codex, Cursor, and others) npx skills add forkadarshp/MPort ``` -*Installing for every agent on your machine? Add `--agent '*'`:* +```text +# Claude Code, as a plugin +/plugin install modelport --marketplace forkadarshp/MPort +``` -```bash -npx skills add forkadarshp/MPort --agent '*' +**2. Ask.** In your project, tell the agent where you are going: + +```text +Migrate this repo to claude-opus-5-5. Keep tool and output behavior the same. ``` +**3. Read the report.** You get the changes, the test and smoke results, the +risks, and the rollback steps. Nothing is called done without evidence. + +### Copy-paste prompts + +| Goal | Prompt | +| :--- | :----- | +| Latest Claude | `Migrate this repo to claude-opus-5-5. Apply required fixes first, then list optional tuning separately.` | +| Cheaper and faster Claude | `Move the classification and routing calls in src/ to claude-haiku-5-5 and keep the output format identical.` | +| GPT-6 | `Migrate our OpenAI calls from gpt-5.5 to gpt-6.1-sol. Move tool calling to the Responses API where needed.` | +| Gemini | `Upgrade our Gemini calls to gemini-3.8-flash and replace thinking_budget with thinking_level.` | +| Switch provider | `Port services/assistant from OpenAI to Claude Sonnet 5.5. Keep the JSON output contract and tool behavior.` | +| Fix a broken swap | `We changed the model ID and now get 400s. Find the cause and fix it.` | +| Prove it | `Migrate to claude-opus-5-5 and benchmark old vs new on our eval set.` | + +More in [examples/example-prompts.md](examples/example-prompts.md). + +## Why not just read the migration guide? + +The provider's guide tells you what changed. You still have to find every +caller, decide the fix order, re-tune prompts, and check nothing regressed. + +| | Provider migration guide | Asking your agent with no skill | ModelPort | +| :--- | :--- | :--- | :--- | +| Knows the current breaking changes | Yes, one provider | Only if it is in its training data | Yes, across providers, dated and sourced | +| Finds every caller, prompt, and tool definition | You do it | Sometimes | Discovery phase, then classification | +| Keeps required fixes apart from tuning | No | Rarely | Always, and labeled | +| Ends with evidence | No | Rarely | Tests, live smoke check, rollback notes | +| Compares old vs new | No | No | Optional three-arm benchmark | + ### Manual Installation (Advanced) If you prefer to fork and modify the skill locally: @@ -199,7 +214,7 @@ mkdir -p ~/.codex/skills/modelport-skill cp -R . ~/.codex/skills/modelport-skill ``` -Then ask Codex: +Then ask your agent: ```text Use $modelport-skill and the starter prompt template to migrate @@ -250,7 +265,7 @@ More examples are in [examples/example-prompts.md](examples/example-prompts.md). ## Expected output -The skill guides Codex to produce a migration report with: +The skill guides your agent to produce a migration report with: - Migration summary - Scope and files changed @@ -269,7 +284,9 @@ See [examples/example-output.md](examples/example-output.md) for a sample report ```text . +├── .claude-plugin/ (Claude Code plugin + marketplace manifests) ├── .markdownlint-cli2.yaml +├── _config.yml (GitHub Pages site config) ├── SKILL.md ├── README.md ├── CONTRIBUTING.md @@ -279,11 +296,14 @@ See [examples/example-output.md](examples/example-output.md) for a sample report ├── assets/ │ └── social-preview.svg ├── scripts/ -│ └── validate_skill.py +│ ├── validate_skill.py +│ └── check_freshness.py +├── harness/ (offline + live old-vs-new benchmark) ├── templates/ │ └── migration-starter-prompt.md ├── references/ │ ├── benchmarking.md +│ ├── breaking-changes.md │ ├── migration-patterns.md │ ├── migration-playbook.md │ ├── output-style.md @@ -291,10 +311,17 @@ See [examples/example-output.md](examples/example-output.md) for a sample report │ ├── provider-checklists.md │ ├── validation-proof.md │ └── models/ -│ ├── claude-opus-4-8.md -│ ├── claude-opus-4-6.md -│ ├── claude-sonnet-4-5.md -│ └── gpt-5-5.md +│ ├── INDEX.md +│ ├── claude-fable-5-1.md +│ ├── claude-opus-5-5.md +│ ├── claude-sonnet-5-5.md +│ ├── claude-haiku-5-5.md +│ ├── gpt-6.md +│ ├── gemini-3-8-flash.md +│ ├── claude-opus-4-8.md (legacy) +│ ├── claude-opus-4-6.md (legacy) +│ ├── claude-sonnet-4-5.md (legacy) +│ └── gpt-5-5.md (legacy) ├── examples/ │ ├── example-prompts.md │ └── example-output.md @@ -305,7 +332,9 @@ See [examples/example-output.md](examples/example-output.md) for a sample report └── .github/ ├── ISSUE_TEMPLATE/ ├── pull_request_template.md - └── workflows/ci.yml + └── workflows/ + ├── ci.yml + └── freshness.yml ``` ## Validate locally @@ -326,30 +355,67 @@ npx markdownlint-cli2 "**/*.md" ## Roadmap -- Per-model migration guides for Claude Opus 4.8 / 4.6, Sonnet 4.5, and GPT-5.5 - ship today in `references/models/`. Next: Gemini and self-hosted models. +- Per-model migration guides for Claude Fable 5.1 / Opus 5.5 / Sonnet 5.5 / + Haiku 5.5, GPT-6, and Gemini 3.8 Flash ship today in `references/models/`, + with legacy Claude and GPT-5.5 guides layered underneath. Next: Gemini Pro + and Flash-Lite guides, and self-hosted models. - Add regression fixtures for prompt, tool, and structured-output migrations. -- Ship a runnable benchmark harness to auto-collect the three-arm leaderboard - (the methodology lands today in `references/benchmarking.md`). +- Add OpenAI and Gemini providers to the benchmark harness (Anthropic and an + offline simulator ship today). - Add real-world before/after migration case studies. ## FAQ -### How do I migrate to Claude Opus 4.8? +### How do I fix "thinking.type.disabled is not supported for this model"? + +Claude Opus 5.5 and Fable 5.1 always think, so `thinking: {type: "disabled"}` +returns a 400. Remove the `thinking` field and set `output_config.effort` to +`low` if you need speed. On Sonnet 5.5, use `{type: "between_tools"}` when a +route must stay thinking-off. Details in the +[Opus 5.5 guide](references/models/claude-opus-5-5.md). + +### How do I fix "tool_choice: type tool and any are not supported for this model"? + +Forced tool use is gone on Claude Opus 5.5, Sonnet 5.5, and Fable 5.1. Use +`tool_choice: {type: "auto"}` with `strict: true` on the tool and a prompt +instruction, then check that a call happened. If the forced call only existed +to get JSON back, use structured outputs. + +### Why are my replies cut off after upgrading Claude? + +Thinking is on by default on the newer models and counts toward `max_tokens`. A +limit sized for a short answer can be used up before any text is written. Raise +`max_tokens` or lower `effort`, and read content blocks by `type` because the +first block may be a `thinking` block. + +### What changes when I move from GPT-5.5 to GPT-6? + +Tool calling moves to the Responses API on Astra and 6.1 Sol, `temperature`, +`top_p`, and `logprobs` go away when reasoning is on, and the `none` and +`minimal` effort levels are not supported on those two models. See the +[GPT-6 guide](references/models/gpt-6.md). + +### What changes when I move to Gemini 3.8 Flash? + +Replace `thinking_budget` with `thinking_level` (sending both returns 400), +drop the `minimal` level, and remove sampling parameters and prefilled model +turns. See the [Gemini 3.8 Flash guide](references/models/gemini-3-8-flash.md). + +### How do I migrate to Claude Opus 5.5? Install ModelPort, then ask your agent: *"migrate this repo to -`claude-opus-4-8`."* It reads the -[Opus 4.8 guide](references/models/claude-opus-4-8.md), applies the required +`claude-opus-5-5`."* It reads the +[Opus 5.5 guide](references/models/claude-opus-5-5.md), applies the required compatibility fixes, preserves tool calling and output contracts, and returns validation evidence plus rollback notes. -### How do I migrate from Claude Opus 4.7 to 4.8? +### How do I migrate from an older Opus (4.x) to the latest? -4.7 → 4.8 is a drop-in upgrade with no breaking API changes — the main task is -re-baselining any tuned `effort` settings. ModelPort makes the model-ID change, -flags the settings to retest, and validates that behavior is unchanged. Coming -from 4.6 or earlier, it applies the 4.7 breaking changes first (sampling params, -adaptive thinking, tokenizer, removed prefills). +Upgrades layer. From 4.7 or older, ModelPort applies the +[Opus 4.8 guide](references/models/claude-opus-4-8.md) first (sampling params, +adaptive thinking, tokenizer, removed prefills), then the Opus 5.5 changes +(thinking always on, forced `tool_choice` removed, preserved thinking, computer +toolset), and re-baselines `effort`, whose default is now `medium`. ### Will changing the model ID break my tool calling or output format? @@ -361,7 +427,7 @@ target model's actual behavior — researched from observed runs, web research, and provider changelogs, including unadvertised quirks — then validates the contract before finishing. -### How do I benchmark Opus 4.8 against my current model? +### How do I benchmark a new model against my current one? Opt into benchmarking at the start and ModelPort runs a three-arm comparison — baseline, naive swap, and the enhanced system — on the same eval set, then @@ -370,14 +436,21 @@ reports a leaderboard with quality, latency, cost, and contract metrics. See ### Does it work with Claude Code, Codex, and Cursor? -Yes. It's a universal agent skill — install it once and invoke it from Claude -Code, Codex, Cursor, or any compatible agent. +Yes. Install it with `npx skills add forkadarshp/MPort` for any compatible +agent, or as a Claude Code plugin with +`/plugin install modelport --marketplace forkadarshp/MPort`. + +### Do I need an API key or an account? + +No. The skill is a set of instructions and guides your agent reads. An API key +is needed only if you run the benchmark harness against a live model. ### Which models and providers does it support? -Any model or provider swap (Anthropic, OpenAI, and others). It ships dedicated -migration guides for Claude Opus 4.8 / 4.6, Claude Sonnet 4.5, and GPT-5.5, and -a generic playbook for everything else. +Any model or provider swap (Anthropic, OpenAI, Google, and others). It ships dedicated +migration guides for Claude Fable 5.1, Opus 5.5, Sonnet 5.5, and Haiku 5.5, +GPT-6, and Gemini 3.8 Flash (plus legacy Claude and GPT-5.5 guides), and a +generic playbook for everything else. ## Contributing diff --git a/SKILL.md b/SKILL.md index 6f0ada4..6411c7a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: modelport-skill -description: Use when the user asks to migrate, upgrade, or port prompts, agents, tools, or codebases from one LLM model or provider version to another. Also use when auditing or continuing migration work started by another agent. Covers API calls, prompt assets, agent configs, tool definitions, tests, docs, runtime settings, rollout plans, and handoff coordination. Can also benchmark pre/post migration systems (old model and prompts vs new model and enhanced system) and report the stats as evidence. +description: Use when the user asks to migrate, upgrade, or port prompts, agents, tools, or code to a newer LLM or a different provider, including Claude Opus 5.5 / Sonnet 5.5 / Haiku 5.5 / Fable 5.1, GPT-6, and Gemini 3.8, or replace a deprecated or retired model. Also use when a model swap returns 400s (thinking disabled, budget_tokens, forced tool_choice, temperature/top_p, prefill, computer_20251124), truncates replies, or breaks parsers and tool calls, and when auditing migration work another agent started. Fixes API calls, prompts, tool definitions, tests, and configs, then proves it with tests, a live smoke check, and an optional old-vs-new benchmark. --- # ModelPort Skill @@ -89,10 +89,19 @@ Collect before editing: report (baseline old/old vs new/enhanced, plus the raw-swap control)? Opt-in. Ask once, up front — capturing the baseline later is impossible. -If source or target matches a model with a dedicated guide in -`references/models/`, read that guide before proceeding. Model guides contain +Look up source and target in `references/models/INDEX.md`. If either has a +dedicated guide, read it before proceeding, and follow the upgrade path in the +index (older guides are layered under newer ones). Model guides contain changelogs, breaking changes, migration steps, and version-specific quirks. +Freshness check, before trusting any guide: + +- `Last verified:` older than 30 days, or the target is not in the index → + fetch the provider's official migration page first. Never guess model IDs, + parameters, prices, or limits. +- Record the source and date used in the migration report. +- Facts marked as secondary-source in a guide → confirm on the provider page. + If benchmarking is requested, read `references/benchmarking.md` and capture the baseline (Arm A) in Phase 0 before any edit — it cannot be reconstructed later. @@ -126,9 +135,12 @@ Read: - `references/migration-playbook.md` - `references/provider-checklists.md` +- `references/models/INDEX.md` — registry, upgrade paths, freshness rules +- `references/breaking-changes.md` — error/symptom → cause → fix lookup across + providers; read first when a swap returns 400s or behaves differently - `references/models/.md` — when source or target matches a model with - a dedicated guide (e.g. `claude-opus-4-8.md`, `claude-opus-4-6.md`, - `claude-sonnet-4-5.md`, `gpt-5-5.md`) + a dedicated guide (Claude Fable 5.1 / Opus 5.5 / Sonnet 5.5 / Haiku 5.5, + GPT-6, Gemini 3.8 Flash, plus legacy Claude and GPT-5.5 guides) - `references/prompt-template-research.md` — when creating or improving a migration prompt or reusable migration request - `references/migration-patterns.md` — when choosing rollout strategy, @@ -249,7 +261,7 @@ Format per `examples/example-output.md`. - Migration strategy: `references/migration-playbook.md` - Provider checklists: `references/provider-checklists.md` - Model-specific guides: `references/models/` (per-version changelogs, quirks, - migration steps) + migration steps). Start at `references/models/INDEX.md`. - Prompt design research: `references/prompt-template-research.md` - Migration patterns: `references/migration-patterns.md` - Validation proof gates: `references/validation-proof.md` diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..d7e4ffa --- /dev/null +++ b/_config.yml @@ -0,0 +1,21 @@ +# GitHub Pages site config. The repo root is the site source, so every guide +# becomes an indexable page. The skill itself does not read this file. +title: ModelPort +description: >- + Upgrade LLM code to Claude Opus 5.5, GPT-6, or Gemini 3.8 without breaking + prod. Breaking changes, fixes, and an agent skill that applies them. +url: https://forkadarshp.github.io +baseurl: /MPort +theme: jekyll-theme-primer +plugins: + - jekyll-seo-tag + - jekyll-sitemap +exclude: + - harness/ + - scripts/ + - templates/ + - agents/ + - docs/PUBLISHING.md + - CODE_OF_CONDUCT.md + - SECURITY.md + - LICENSE diff --git a/agents/openai.yaml b/agents/openai.yaml index 45009fe..3a60325 100644 --- a/agents/openai.yaml +++ b/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "ModelPort" - short_description: "Migrate prompts, agents, and LLM code safely" - default_prompt: "Use $modelport-skill and the starter prompt template to migrate this codebase from the current model to the newer target model with validation evidence and rollback notes." + short_description: "Upgrade to newer LLMs (Claude 5.x, GPT-6, Gemini 3.8) without breaking prod" + default_prompt: "Use $modelport-skill to migrate this codebase to the target model. Read references/models/INDEX.md, apply required compatibility fixes before optional tuning, and return validation evidence plus rollback notes." diff --git a/assets/social-preview.png b/assets/social-preview.png index c9ea9ef..92a0601 100644 Binary files a/assets/social-preview.png and b/assets/social-preview.png differ diff --git a/assets/social-preview.svg b/assets/social-preview.svg index a2dec2c..cd4004c 100644 --- a/assets/social-preview.svg +++ b/assets/social-preview.svg @@ -1,31 +1,26 @@ ModelPort - Safe LLM upgrades for prompts, agents, and production code. + Upgrade to Claude Opus 5.5, GPT-6, or Gemini 3.8 without breaking prod. An agent skill that fixes the breaking changes and proves the upgrade. - - + + __ __ _ _ _____ _ - | \/ | | | | | __ \ | | - | \ / | ___ __| | ___| | |__) |__ _ __| |_ - | |\/| |/ _ \ / _` |/ _ \ | ___/ _ \| '__| __| - | | | | (_) | (_| | __/ | | | (_) | | | |_ - |_| |_|\___/ \__,_|\___|_|_| \___/|_| \__| + | \/ | | | | | __ \ | | + | \ / | ___ __| | ___| | |__) |__ _ __| |_ + | |\/| |/ _ \ / _` |/ _ \ | ___/ _ \| '__| __| + | | | | (_) | (_| | __/ | | | (_) | | | |_ + |_| |_|\___/ \__,_|\___|_|_| \___/|_| \__| - -+------------------------------------------------------------------------------------------+ -| | -| > Safe LLM upgrades for prompts, agents, and production code. | -| | -| | -| [ Legacy Provider ] ==========> ( ModelPort Engine ) ==========> [ Next-Gen Model ] | -| | -| | -| > init migration_protocol .................................................... [ OK ] | -| > mapping caller routes ...................................................... [ OK ] | -| > applying surgical code patches ............................................. [ OK ] | -| > running regression evals ................................................... [ OK ] | -| | -+------------------------------------------------------------------------------------------+ + Upgrade models without breaking prod. + An agent skill for Claude Opus 5.5, GPT-6 and Gemini 3.8 migrations. + + +x "thinking.type.disabled" is not supported for this model +x tool_choice: type "tool" and "any" are not supported +x reply truncated: thinking counts toward max_tokens +> fixed in order, tested, with rollback notes + + npx skills add forkadarshp/MPort diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 0cf581c..46dd748 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -1,53 +1,183 @@ -# GitHub Publishing Checklist +# Publishing and Growth Playbook -Use these settings before making the repository public. +Maintainer checklist for repo settings, release cadence, and distribution. -## Repository description +## Positioning + +- **One line:** Upgrade models without breaking prod. +- **Who it is for:** engineers whose app calls an LLM API and who must move to + a newer model (a deprecation notice, a price drop, a capability they need). + Secondary: teams that use Claude Code, Codex, or Cursor and want an agent to + do the migration. +- **Why it wins:** per-model guides with exact breaking changes and error + strings, a fix order, and proof. A generic "migrate my prompts" request to an + agent has none of that. +- **Evergreen, not launch-tied.** Do not build the pitch on one model release. + The pitch is "every release breaks something". The model list is a table that + gets a new row. + +## Repository settings + +Set these in the GitHub repo settings (or with `gh repo edit`). + +Description (under 160 characters): ```text -Codex skill for safe LLM model migrations across prompts, agents, API callers, tests, rollout plans, and rollback paths. +Agent skill that upgrades LLM code to Claude Opus 5.5, GPT-6, Gemini 3.8 without breaking prod. Fixes breaking changes, then proves it with tests and evals. ``` -## Recommended GitHub topics +Topics (20 is the maximum): ```text -llm -model-migration -prompt-engineering -ai-agents -codex -openai -anthropic -gemini -developer-tools -llmops -prompt-migration -model-upgrade +agent-skills, claude-code, codex, cursor, llm-migration, model-migration, +model-upgrade, prompt-migration, prompt-engineering, llmops, llm, anthropic, +openai, gemini, claude, gpt, ai-agents, developer-tools, breaking-changes, +benchmarking ``` -## Social preview +Avoid version-specific topics (for example a model ID). They go stale. -```text -Headline: ModelPort -Subhead: Safe LLM upgrades for prompts, agents, and production code -Visual: old model -> compatibility map -> validated rollout +Social preview: upload `assets/social-preview.png` under Settings, Social +preview. GitHub does not accept this through the API. Regenerate the PNG after +editing the SVG: + +```bash +rsvg-convert -w 1280 -h 640 assets/social-preview.svg -o assets/social-preview.png ``` -Use `assets/social-preview.svg` as the starting social preview artwork. +## Release cadence + +Model news drives search traffic, so speed matters more than polish. + +When a provider ships a model: + +1. Within 48 hours, add a guide in `references/models/`. Copy the header block + of an existing guide: `Model ID`, price, `Last verified`, `Source`. +2. Add the guide to `references/models/INDEX.md` and, if the model brings new + errors, to `references/breaking-changes.md`. +3. Add a row to the README "Supported models" table. +4. Run `python3 scripts/validate_skill.py .`. +5. Tag a release. Use the error strings and the one-line fix in the notes. + +The weekly `freshness` workflow opens an issue when any guide passes 30 days +without re-verification. It cannot see new model releases, so also check the +provider changelogs when you re-verify: + +- Anthropic: +- OpenAI: +- Google: + +Version history: + +- `v0.1.0`: first public release (workflow, references, examples, CI). +- `v0.2.0`: current-generation guides (Claude 5.x, GPT-6, Gemini 3.8), model + index, breaking-changes lookup, freshness automation, harness fixes for + current Claude models. +- `v1.0.0`: stable workflow validated on real migrations (case studies). + +## Install channels + +Keep both working. Test them after any change to `SKILL.md` or the manifests. + +| Channel | User command | Check | +| :------ | :----------- | :---- | +| Skills CLI (any agent) | `npx skills add forkadarshp/MPort` | `npx skills add . --list` finds one skill | +| Claude Code plugin | `/plugin install modelport --marketplace forkadarshp/MPort` | `claude plugin validate .` passes | + +Listing on skills.sh: . +On 2026-10-11 it showed 1 install and "No SKILL.md available for this skill", +so the page had no description. The raw `SKILL.md` URL resolves, so the cause is +on the directory side or a stale snapshot. Re-check after the next release and +report it to the skills.sh maintainers if it persists. + +Wider reach for the plugin: submit it to Anthropic's plugin directory (see +"Submit to Anthropic's directory" in the Claude Code plugin docs). + +## Website (GitHub Pages) -## First-screen README copy +`_config.yml` makes the repo root a Jekyll site, so every guide becomes its own +indexable page with a title taken from its first heading. Pages is off until +you enable it: Settings, Pages, deploy from branch `main`, folder `/`. Then set +the repo homepage to the site URL and confirm the build passed under Actions. +The config has not been built locally, so check the first build. + +## Naming + +The repo is `MPort`, the product is ModelPort, and the skill is +`modelport-skill`. Nobody searches for "MPort". Renaming the repo to +`modelport` puts the search term in the URL and in the install command. GitHub +redirects the old URL, but update the README, the manifests, `_config.yml`, and +this file in the same change. + +## Distribution + +`npx skills add` installs feed the public skills directory at skills.sh, so +installs and discovery reinforce each other. Channels, in order of expected +return: + +1. **Search.** The breaking-changes page contains exact error strings people + paste into search. Write one blog post or README-linked page per major + model release: "What breaks when you move to MODEL NAME" with the error, + cause, and fix. Link back to the repo. +2. **Show HN and Reddit** (r/ClaudeAI, r/ChatGPTCoding). Post once per major + release, with the breaking-changes table as the hook, not the tool. +3. **X or LinkedIn thread** on the same table. +4. **Provider communities** (Anthropic and OpenAI developer Discords and + forums) as an answer to a real migration question, with the relevant guide + linked. +5. **Awesome lists**, via pull request. Read each list's contribution rules + first. Candidates: `hesreallyhim/awesome-claude-code`, + `travisvn/awesome-claude-skills`, `ComposioHQ/awesome-claude-skills`, + `karanb192/awesome-claude-skills`, `VoltAgent/awesome-agent-skills`. + +Before posting, check each community's self-promotion rules. + +### Draft: Show HN + +```text +Title: Show HN: ModelPort, an agent skill that fixes what breaks when you upgrade LLMs + +Moving to Claude Opus 5.5 means thinking can no longer be disabled, forced +tool_choice returns 400, and thinking blocks must stay append-only. GPT-6 +drops temperature and top_p when reasoning is on. Gemini 3.8 rejects the +"minimal" thinking level. None of this is a find/replace. + +ModelPort is a skill for Claude Code, Codex and Cursor. It has per-model guides +(Claude 5.x, GPT-6, Gemini 3.8) with the exact errors and fixes, applies them in +order, and ends with tests, a live smoke check and rollback notes. There is also +an offline harness for old-vs-new comparisons. The simulator's numbers are +illustrative; --provider anthropic gives measured ones. + +Guides carry a verified-on date and a source, and CI flags stale ones. + +https://github.com/forkadarshp/MPort +``` + +### Draft: thread ```text -Most model migrations fail after the model ID changes. +1/ Upgrading an LLM is never just the model ID. Things I keep seeing break: +- "thinking.type.disabled" is not supported (Claude Opus 5.5) +- tool_choice any/tool returns 400 (Opus 5.5, Sonnet 5.5, Fable 5.1) +- temperature/top_p rejected with reasoning on (GPT-6) +- replies cut off because thinking now counts toward max_tokens + +2/ I put every one of these, with the cause and the fix, in one table: +references/breaking-changes.md -ModelPort treats migration as a behavior-preservation problem, not a -search-and-replace task. It separates required compatibility fixes from optional -prompt and agent tuning, then ends every migration with validation evidence and -rollback notes. +3/ ModelPort is an agent skill that applies those fixes for you, in order, and +ends with tests, a smoke check and rollback notes. npx skills add forkadarshp/MPort ``` -## Release tags +## Measure + +Check weekly: stars, unique visitors and referrers (Insights, Traffic), clones, +and issues opened. Referrers show which channel worked. Repeat what works. + +## Before making a release public -- `v0.1.0`: first public release with skill workflow, references, examples, and CI. -- `v0.2.0`: provider deep dives and migration regression fixtures. -- `v1.0.0`: stable migration workflow with validated examples across providers. +- [ ] `python3 scripts/validate_skill.py .` passes. +- [ ] `npx markdownlint-cli2 "**/*.md"` passes. +- [ ] `python3 -m unittest discover -s harness/tests` passes. +- [ ] README "Supported models" table matches `references/models/INDEX.md`. +- [ ] Description, topics, and social preview match this file. diff --git a/examples/example-prompts.md b/examples/example-prompts.md index 24493aa..8158c77 100644 --- a/examples/example-prompts.md +++ b/examples/example-prompts.md @@ -13,8 +13,17 @@ Use these to test ModelPort triggering and behavior. 7. "Migrate this repo to the latest model family and separate required fixes vs optional tuning." 8. "Upgrade model references across config, tests, and docs without breaking runtime routing." +## Current-model migrations + +1. "Migrate this repo to `claude-opus-5-5`. Apply required fixes first, then list optional tuning separately." +2. "Move the classification and routing calls in `src/` to `claude-haiku-5-5` and keep the output format identical." +3. "Migrate our OpenAI calls from `gpt-5.5` to `gpt-6.1-sol`. Move tool calling to the Responses API where needed." +4. "Upgrade our Gemini calls to `gemini-3.8-flash` and replace `thinking_budget` with `thinking_level`." +5. "We changed the model ID and now get 400s. Find the cause and fix it." +6. "Our agent UI goes quiet between tool calls since the upgrade. Find out why and fix it." + ## Advanced migrations -1. "Migrate from OpenAI `gpt-4o` to Anthropic `claude-sonnet-4-20250514` across all agents, normalize output contracts, and add provider-agnostic wrapper layers." +1. "Migrate from OpenAI `gpt-5.5` to Anthropic `claude-sonnet-5-5` across all agents, normalize output contracts, and add provider-agnostic wrapper layers." 2. "Agent B already started migrating `services/chat` — pick up where it left off, verify what's done, and finish the remaining work with validation." 3. "Migrate our structured output pipeline from JSON mode to native structured outputs on the new model, update Zod schemas, and regression-test all tool call paths." diff --git a/harness/README.md b/harness/README.md index 2c841a8..68cdb2e 100644 --- a/harness/README.md +++ b/harness/README.md @@ -70,13 +70,20 @@ slightly. Two real takeaways the harness surfaces: Opus 4.7+ trait: newer models follow instructions more literally, punishing vague prompts and rewarding precise ones). The grading, scoring, and leaderboard pipeline is real; **the numbers are illustrative, not measured.** -- **`anthropic`** — real Messages API calls. Same graders, real numbers. +- **`anthropic`** — real Messages API calls. Same graders, real numbers. Safe + for current models: it never sends `temperature`, `thinking`, prefill, or a + forced `tool_choice` (all rejected by Opus 5.5, Sonnet 5.5, Haiku 5.5, and + Fable 5.1), reads content blocks by type, and treats a `refusal` stop reason + as a failed case. Add an optional `request` block to a scenario to set + `effort` and `max_tokens` (default 4096, which leaves room for always-on + thinking). ## Define your own scenario Copy `scenarios/support_triage.json` and edit: - `models` — old/new model IDs +- `request` — optional `{"effort": "low", "max_tokens": 4096}` for `--provider anthropic` - `prompts.baseline` / `prompts.enhanced` — the configs for arms A/B vs C - `tools`, `categories` — the task surface - `eval_cases` — inputs + expected `{category, tool, args}` and a `difficulty` diff --git a/harness/providers.py b/harness/providers.py index dc51870..9232321 100644 --- a/harness/providers.py +++ b/harness/providers.py @@ -16,10 +16,14 @@ from __future__ import annotations import json +import sys import time from graders import Response +# Leaves headroom for always-on thinking on current Claude models. +DEFAULT_MAX_TOKENS = 4096 + # Simulator formula constants (transparent, tunable). See module docstring. _BOOST_EXPLICIT = 0.30 # precise prompt raises fidelity _PENALTY_VAGUE = 0.20 # literal model * vague prompt lowers fidelity @@ -145,23 +149,45 @@ def generate(self, model: str, system_prompt: str, case: dict) -> Response: for t in self.scenario["tools"] if t["name"] != "none" ] + # Optional per-scenario request settings. Newer models (Opus 5.5, + # Haiku 5.5, Fable 5.1) always think and thinking counts toward + # max_tokens, so the default leaves room for it. Sampling parameters, + # prefill, forced tool_choice, and thinking config are deliberately + # never sent: current models reject them. + request = self.scenario.get("request", {}) + kwargs = {} + if request.get("effort"): + kwargs["output_config"] = {"effort": request["effort"]} + start = time.time() msg = self.client.messages.create( model=model, - max_tokens=512, + max_tokens=request.get("max_tokens", DEFAULT_MAX_TOKENS), system=system_prompt, tools=tools, messages=[{"role": "user", "content": case["input"]}], + **kwargs, ) latency_ms = (time.time() - start) * 1000 + # Classifier declines arrive as HTTP 200 with stop_reason "refusal". + # Grade them as an empty response instead of reading content[0]. + if msg.stop_reason == "max_tokens": + print( + f"warning: {model} hit max_tokens={request.get('max_tokens', DEFAULT_MAX_TOKENS)}; " + "raise scenario request.max_tokens or lower request.effort", + file=sys.stderr, + ) + text, tool_name, tool_args = "", None, {} - for block in msg.content: - if block.type == "text": - text += block.text - elif block.type == "tool_use": - tool_name = block.name - tool_args = dict(block.input) + if msg.stop_reason != "refusal": + # Read blocks by type: a response can start with thinking blocks. + for block in msg.content: + if block.type == "text": + text += block.text + elif block.type == "tool_use": + tool_name = block.name + tool_args = dict(block.input) return Response( text=text, diff --git a/harness/scenarios/ops_routing.json b/harness/scenarios/ops_routing.json index 2c4403e..b3a9c34 100644 --- a/harness/scenarios/ops_routing.json +++ b/harness/scenarios/ops_routing.json @@ -2,7 +2,7 @@ "name": "ops-tool-routing", "role": "You are an ops-assistant that routes a request to exactly one tool.", "description": "Route a natural-language ops request to one of six tools with the right multi-field args, and return strict JSON. Harder than triage: more tools (more ways to mis-route), multi-arg calls, and several compound/ambiguous cases that prompting alone can't fully resolve.", - "models": { "old": "claude-opus-4-6", "new": "claude-opus-4-8" }, + "models": { "old": "claude-opus-4-6", "new": "claude-opus-5-5" }, "categories": ["deploy", "reliability", "observability", "incident", "other"], "tools": [ { "name": "restart_service", "args": ["service"] }, @@ -34,6 +34,6 @@ ], "sim": { "claude-opus-4-6": { "literalness": 0.5, "base": 0.66, "latency_ms": 3800, "price_in": 0.000005, "price_out": 0.000025 }, - "claude-opus-4-8": { "literalness": 0.9, "base": 0.70, "latency_ms": 2300, "price_in": 0.000005, "price_out": 0.000025 } + "claude-opus-5-5": { "literalness": 0.9, "base": 0.70, "latency_ms": 2300, "price_in": 0.000004, "price_out": 0.00002 } } } diff --git a/harness/scenarios/support_triage.json b/harness/scenarios/support_triage.json index 6b92b27..e0a9dc0 100644 --- a/harness/scenarios/support_triage.json +++ b/harness/scenarios/support_triage.json @@ -2,7 +2,7 @@ "name": "support-ticket-triage", "role": "You are a support-triage assistant.", "description": "Classify a support ticket, optionally call one tool, and return a strict JSON object. Exercises output-contract conformance and tool-calling accuracy across a model migration.", - "models": { "old": "claude-opus-4-6", "new": "claude-opus-4-8" }, + "models": { "old": "claude-opus-4-6", "new": "claude-opus-5-5" }, "categories": ["billing", "technical", "account", "other"], "tools": [ { "name": "lookup_order", "args": ["order_id"] }, @@ -32,6 +32,6 @@ ], "sim": { "claude-opus-4-6": { "literalness": 0.5, "base": 0.72, "latency_ms": 3800, "price_in": 0.000005, "price_out": 0.000025 }, - "claude-opus-4-8": { "literalness": 0.9, "base": 0.76, "latency_ms": 2300, "price_in": 0.000005, "price_out": 0.000025 } + "claude-opus-5-5": { "literalness": 0.9, "base": 0.76, "latency_ms": 2300, "price_in": 0.000004, "price_out": 0.00002 } } } diff --git a/harness/tests/test_harness.py b/harness/tests/test_harness.py index d953361..d4045c9 100644 --- a/harness/tests/test_harness.py +++ b/harness/tests/test_harness.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(HARNESS)) from graders import Response, grade_contract, grade_task, grade_tool # noqa: E402 -from providers import SimProvider, explicitness # noqa: E402 +from providers import DEFAULT_MAX_TOKENS, AnthropicProvider, SimProvider, explicitness # noqa: E402 import run # noqa: E402 @@ -81,3 +81,70 @@ def test_modelport_arm_beats_baseline(self): if __name__ == "__main__": unittest.main() + + +class _Block: + def __init__(self, type, **kw): + self.type = type + self.__dict__.update(kw) + + +class _Msg: + def __init__(self, content, stop_reason="end_turn"): + self.content = content + self.stop_reason = stop_reason + self.usage = type("U", (), {"input_tokens": 10, "output_tokens": 5})() + + +class _FakeClient: + """Records the request and returns a canned message (no network).""" + + def __init__(self, msg): + self.calls = [] + self.messages = self + self._msg = msg + + def create(self, **kwargs): + self.calls.append(kwargs) + return self._msg + + +def _provider(scenario_extra, msg): + prov = AnthropicProvider.__new__(AnthropicProvider) # skip the anthropic import + prov.scenario = {"tools": [{"name": "lookup", "args": ["id"]}], **scenario_extra} + prov.client = _FakeClient(msg) + return prov + + +class TestAnthropicProviderNewModels(unittest.TestCase): + CASE = {"input": "where is order 4471?"} + + def test_request_avoids_params_current_models_reject(self): + prov = _provider({}, _Msg([_Block("text", text="{}")])) + prov.generate("claude-opus-5-5", "sys", self.CASE) + sent = prov.client.calls[0] + for banned in ("temperature", "top_p", "top_k", "thinking", "tool_choice"): + self.assertNotIn(banned, sent) + self.assertEqual(sent["max_tokens"], DEFAULT_MAX_TOKENS) + self.assertEqual(sent["messages"][-1]["role"], "user") # no prefill + + def test_effort_is_sent_only_when_configured(self): + prov = _provider({"request": {"effort": "low", "max_tokens": 2048}}, + _Msg([_Block("text", text="{}")])) + prov.generate("claude-haiku-5-5", "sys", self.CASE) + sent = prov.client.calls[0] + self.assertEqual(sent["output_config"], {"effort": "low"}) + self.assertEqual(sent["max_tokens"], 2048) + + def test_reads_blocks_by_type_after_leading_thinking_block(self): + msg = _Msg([_Block("thinking", thinking=""), + _Block("tool_use", name="lookup", input={"id": "4471"})]) + resp = _provider({}, msg).generate("claude-opus-5-5", "sys", self.CASE) + self.assertEqual(resp.tool_name, "lookup") + self.assertEqual(resp.tool_args, {"id": "4471"}) + + def test_refusal_grades_as_empty_response(self): + resp = _provider({}, _Msg([], stop_reason="refusal")).generate( + "claude-opus-5-5", "sys", self.CASE) + self.assertEqual(resp.text, "") + self.assertIsNone(resp.tool_name) diff --git a/references/breaking-changes.md b/references/breaking-changes.md new file mode 100644 index 0000000..9341264 --- /dev/null +++ b/references/breaking-changes.md @@ -0,0 +1,50 @@ +# LLM upgrade errors and fixes: Claude 5.x, GPT-6, Gemini 3.8 + +Symptom to cause to fix, across providers. Use it when a model swap starts +returning errors or behaving differently. Each row points at the guide that +has the full migration. Facts verified 2026-10-11 against provider docs (see +each guide's `Source:` line). Exact error text is quoted where the provider +documents it. Where it is not documented here, the symptom is described +instead of guessing a string. + +## Request errors (HTTP 400) + +| You send | Fails on | Error or symptom | Fix | +| :------- | :------- | :--------------- | :-- | +| `thinking: {type: "disabled"}` | Claude Opus 5.5, Fable 5.1, Fable 5 | `"thinking.type.disabled" is not supported for this model. Use "thinking.type.adaptive" and "output_config.effort" to control thinking behavior.` | Omit `thinking`. Set `output_config.effort` to `low` for speed. | +| `thinking: {type: "disabled"}` | Claude Sonnet 5.5 | `"thinking.type.disabled" is not supported for this model. Use "thinking.type.between_tools" for the lowest thinking setting, or ...` | Try adaptive + `effort: "low"`. Else `{type: "between_tools"}` at effort `high` or below. | +| `thinking: {type: "enabled", budget_tokens: N}` | Claude Opus 4.7 and later, Sonnet 5 and later, Haiku 5.5, Fable | 400 | `{type: "adaptive"}` + `output_config.effort`. | +| `thinking: {type: "disabled"}` with effort `xhigh` or `max` | Claude Opus 5, Haiku 5.5 | `output_config.effort 'xhigh' is not supported when thinking is disabled on this model. Use effort 'high' or below, or enable thinking.` | Lower effort or enable thinking. | +| `thinking: {type: "between_tools"}` | Any model except Claude Sonnet 5.5 | `"thinking.type.between_tools" is not supported for this model.` | Drop the field when re-sending to another model. | +| `tool_choice: {type: "any"}` or `{type: "tool", ...}` | Claude Opus 5.5, Sonnet 5.5, Fable 5.1 | `tool_choice: type "tool" and "any" are not supported for this model.` | `auto` + `strict: true` + prompt instruction, or `output_config.format` for JSON. Check a call happened. | +| `computer_20251124` tool | Claude Opus 5.5, Sonnet 5.5 on Claude API and Google Cloud | `'claude-opus-5-5' does not support tool types: computer_20251124.` | `{"type": "computer_toolset_20260801"}`. Amazon Bedrock still accepts the old tool. | +| `computer_20250124` tool | Claude Haiku 5.5 on Claude API and Google Cloud | 400 | `computer_toolset_20260801`. | +| `temperature`, `top_p`, `top_k` (non-default) | Claude Opus 4.7 and later, Sonnet 5 and later, Haiku 5.5, Fable | 400 | Omit them. Haiku 5.5 accepts only `temperature` 1 and `top_p` 0.99. | +| Assistant prefill (last message is `assistant`) | Claude Opus 4.6 and later, Sonnet 4.6 and later, Haiku 5.5, Fable | 400 | End with a user turn. Use `output_config.format` or a prompt instruction. | +| Replayed thinking block after editing `system`, `tools`, or earlier messages | Claude Fable 5.1, Opus 5.5, Sonnet 5.5, Haiku 5.5 (accounts created on or after 2026-08-31) | 400 | Keep history append-only. See `claude-fable-5-1.md`. | +| `fallbacks` on Haiku 5.5 | Claude Haiku 5.5 | A model list returns 400. `"default"` leaves a decline declined. | Handle the refusal in the client. | +| `thinking_budget` and `thinking_level` together | Gemini 3.x | 400 | Send only `thinking_level`. | +| `thinking_level: "minimal"` | Gemini 3.8 Flash | Error | Use `low`, `medium`, or `high`. | +| `reasoning.effort: "none"` or `"minimal"` | GPT-6 Astra, GPT-6.1 Sol | Unsupported | Use `low`. | +| `temperature`, `top_p`, `top_logprobs`, `logprobs` with reasoning on | GPT-6 family | Unsupported | Remove them. | +| Tool calling on Chat Completions | GPT-6 Astra, 6.1 Sol | Unsupported | Use the Responses API. | + +## Silent changes (no error, output differs) + +| Change | Models | Effect | Fix | +| :----- | :----- | :----- | :-- | +| Thinking on by default | Claude Opus 5, Sonnet 5, Haiku 5.5 | `max_tokens` shared with thinking. Replies truncate. `content[0]` may be a `thinking` block. | Raise `max_tokens` (64K for long agent turns). Read blocks by `type`. | +| Default effort `medium` | Claude Opus 5.5, Haiku 5.5 | Less thinking than the old `high` default. | Set `effort` explicitly. Re-run the sweep. | +| `thinking.display` default `"omitted"` | Claude Opus 4.7 and later, Sonnet 5 and later, Fable | Thinking text is empty. UI shows a long pause. | `display: "summarized"`. | +| Progress notes moved into `thinking` blocks | Claude Opus 5.5, Sonnet 5.5, Fable 5.1 | UI goes quiet between tool calls. | `display: "updates"` (beta) and render non-empty `thinking` blocks. | +| New tokenizer, about 30% more tokens | Claude Sonnet 5 and later (vs 4.6), Haiku 5.5 (vs 4.5) | Cost and context budgets shift. | Recount with `count_tokens`. | +| Classifier refusals (HTTP 200, `stop_reason: "refusal"`) | Claude Fable 5.1, Opus 5 and 5.5, Sonnet 5.5, Haiku 5.5 | Code reading `content[0]` breaks. | Branch on `stop_reason`. Add a fallback where supported. | +| More literal instruction following | Claude Opus 4.7 and later | Vague prompts behave differently. | State format, verbosity, and scope. | +| More tool-call iteration, smaller reasoning steps | Gemini 3.8 Flash | More tokens on long tasks. | Lower `thinking_level`. | +| More clarifying questions, long formatted answers | GPT-6 Astra | Stalls on autonomous work, verbose output. | Prompt for action and style. See `gpt-6.md`. | + +## Where to go next + +- Model-specific steps: `models/INDEX.md`. +- Rollout and rollback: `migration-playbook.md`, `migration-patterns.md`. +- Proving it worked: `validation-proof.md`, `benchmarking.md`. diff --git a/references/models/INDEX.md b/references/models/INDEX.md new file mode 100644 index 0000000..d3d96ed --- /dev/null +++ b/references/models/INDEX.md @@ -0,0 +1,74 @@ +# Model Guide Index + +Registry of dedicated model guides. Read this first: find the source and +target model, open the guide, then follow its upgrade path. + +Status values: **current** (default target), **active** (still served, not the +default target), **legacy** (superseded guide kept for layered upgrades). + +## Anthropic + +| Model | Model ID | Guide | Status | +| :---- | :------- | :---- | :----- | +| Claude Fable 5.1 | `claude-fable-5-1` | [claude-fable-5-1.md](claude-fable-5-1.md) | current (opt-in only) | +| Claude Opus 5.5 | `claude-opus-5-5` | [claude-opus-5-5.md](claude-opus-5-5.md) | current (default Opus target) | +| Claude Sonnet 5.5 | `claude-sonnet-5-5` | [claude-sonnet-5-5.md](claude-sonnet-5-5.md) | current | +| Claude Haiku 5.5 | `claude-haiku-5-5` | [claude-haiku-5-5.md](claude-haiku-5-5.md) | current | +| Claude Opus 4.8 | `claude-opus-4-8` | [claude-opus-4-8.md](claude-opus-4-8.md) | legacy (layer under Opus 5.5) | +| Claude Opus 4.6 | `claude-opus-4-6` | [claude-opus-4-6.md](claude-opus-4-6.md) | legacy | +| Claude Sonnet 4.5 | `claude-sonnet-4-5` | [claude-sonnet-4-5.md](claude-sonnet-4-5.md) | legacy | + +Claude upgrade paths. Apply each step in order, then the target guide. + +| If you are on | Target | Path | +| :------------ | :----- | :--- | +| Opus 4.x or older | `claude-opus-5-5` | `claude-opus-4-6.md`, `claude-opus-4-8.md`, then `claude-opus-5-5.md` | +| Opus 5 | `claude-opus-5-5` | `claude-opus-5-5.md` | +| Sonnet 4.x or older | `claude-sonnet-5-5` | `claude-sonnet-4-5.md`, then `claude-sonnet-5-5.md` | +| Sonnet 5 | `claude-sonnet-5-5` | `claude-sonnet-5-5.md` | +| Haiku 4.5 or older | `claude-haiku-5-5` | `claude-haiku-5-5.md` | +| Fable 5 | `claude-fable-5-1` | `claude-fable-5-1.md` | + +Default target when the user says "latest Claude model" and names no tier: +`claude-opus-5-5`. + +## OpenAI + +| Model | Model ID | Guide | Status | +| :---- | :------- | :---- | :----- | +| GPT-6 Astra, GPT-6.1 Sol, GPT-6 Sol, GPT-6 Luna | `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-sol`, `gpt-6-luna` | [gpt-6.md](gpt-6.md) | current | +| GPT-5.5 | `gpt-5.5` | [gpt-5-5.md](gpt-5-5.md) | legacy (layer under GPT-6) | + +## Google + +| Model | Model ID | Guide | Status | +| :---- | :------- | :---- | :----- | +| Gemini 3.8 Flash | `gemini-3.8-flash` | [gemini-3-8-flash.md](gemini-3-8-flash.md) | current | + +## Not covered + +No dedicated guide: other providers, open-weight models, older Gemini +releases. Use `../provider-checklists.md`, then research (see "Freshness" +below). + +## Freshness + +Every guide carries a `Last verified:` date and a `Source:` line. Model facts +(IDs, prices, defaults, deprecations) change faster than guides do. + +- Guide verified within 30 days of today: trust it, still smoke test. +- Older than 30 days, or the target model is missing from this index: fetch + the provider's official migration page before editing. Record the page and + date in the migration report. +- Facts from secondary sources are marked as such in the guide. Confirm them + on the provider page before quoting prices or dates to the user. +- After refreshing a guide, update its `Last verified:` line and this index. + +Official sources: + +- Anthropic: [models overview](https://platform.claude.com/docs/en/about-claude/models/overview) + and [model deprecations](https://platform.claude.com/docs/en/about-claude/model-deprecations) +- OpenAI: [Using GPT-6](https://developers.openai.com/api/docs/guides/latest-model) + and the [models page](https://developers.openai.com/api/docs/models) +- Google: [Gemini models](https://ai.google.dev/gemini-api/docs/models) and the + [Gemini API changelog](https://ai.google.dev/gemini-api/docs/changelog) diff --git a/references/models/claude-fable-5-1.md b/references/models/claude-fable-5-1.md new file mode 100644 index 0000000..bb10cec --- /dev/null +++ b/references/models/claude-fable-5-1.md @@ -0,0 +1,88 @@ +# Migrating to Claude Fable 5.1: breaking changes and fixes + +What breaks when you move code to `claude-fable-5-1`, the fix for each, and what to re-tune. + +Model ID: `claude-fable-5-1` (Project Glasswing counterpart: +`claude-mythos-5-1`) +Pricing: $10 / $50 per million input / output tokens +Cache reads: $0.25 per million tokens +Context / output: 1M tokens / 128K tokens +Last verified: 2026-10-11 +Source: Anthropic API migration guide (bundled `claude-api` skill, cached +2026-10-06) and platform docs + +Anthropic's most capable widely released model, for the most demanding +reasoning and long-horizon agentic work. **Migrate here only when the user +explicitly chose it.** The default upgrade target for "latest model" is +`claude-opus-5-5`. Fable pricing is above Opus tier. + +## Changelog (vs Fable 5) + +- Forced `tool_choice` (`any`, `tool`) removed. +- Thinking blocks are bound to the producing model. +- Editing earlier turns invalidates thinking blocks (preserved thinking). +- Per-message `effort`, turn-scoped `clear_at: "next_user_message"` system + messages, `thinking.display: "updates"`, cache reads at $0.25 per million. +- Same tokenizer as Opus 4.8 and Fable 5. + +## Required changes (will 400) + +| Change | Detail | +| :----- | :----- | +| Thinking config | Thinking is always on. Omit `thinking` or send `{type: "adaptive"}`. `{type: "disabled"}` and `budget_tokens` both 400. | +| Sampling parameters | Non-default `temperature`, `top_p`, `top_k` 400. | +| Assistant prefill | 400. Use `output_config.format` or system prompt instructions. | +| Forced tool use | `tool_choice` `any` and `tool` 400. Use `auto` + `strict: true`, or structured outputs. | +| Preserved thinking | Replayed thinking blocks after an edit to `system`, `tools`, or earlier `messages` 400 on accounts created on or after 2026-08-31. Keep histories append-only: mid-conversation `role: "system"` messages, `tool_addition` / `tool_removal` blocks, server-side compaction or context editing. | +| Data retention | Requires 30-day retention. Zero-data-retention orgs get `400 invalid_request_error` unless Anthropic expressly authorized it. | +| Refusals | Classifiers can decline cyber, biology research, and similar requests. Check `stop_reason == "refusal"` before reading `content`. | + +## Refusal fallback (ship from day one) + +Fallbacks are not automatic. Add one, tell the user, and drop it only if they +decline. + +| Platform | Mechanism | +| :------- | :-------- | +| Claude API, Claude Platform on AWS | Server-side: beta `server-side-fallback-2026-07-01` + `fallbacks: "default"` (routes by refusal category). Older array form: beta `server-side-fallback-2026-06-01` + `fallbacks: [{"model": "claude-opus-4-8"}]`. | +| Amazon Bedrock, Vertex AI, Foundry | SDK client middleware (`BetaRefusalFallbackMiddleware` + one `BetaFallbackState` per conversation). | +| Raw HTTP | Retry on another model. Use fallback credit (`fallback-credit-2026-07-01`) to keep cache-read pricing. The retry body must match the refused request exactly. | + +Thinking blocks from Fable 5.1 are read only by Fable 5.1 and Mythos 5.1. +After a fallback, the next model runs without that reasoning. Do not strip +blocks yourself. + +## Silent changes + +| Change | Action | +| :----- | :----- | +| `thinking.display` default `"omitted"` | Set `"summarized"` for readable reasoning. Pass blocks back unchanged, empty ones included. | +| Turns run many minutes | Plan timeouts, streaming, and progress UX. Check in asynchronously. | +| Tokenizer | Roughly unchanged from Opus 4.7 and 4.8. From Opus 4.6, Sonnet, or Haiku, expect about 1x to 1.35x more tokens. Re-baseline with `count_tokens`. | +| Effort | Supports `low` to `max`. Lower levels often beat prior-generation `xhigh`. Sweep `low` and `medium` for routine work. | + +## Prompt and behavior tuning (optional) + +- De-prescribe migrated prompts and skills. Step-by-step scaffolding written + for older models often lowers output quality. State goals and constraints. +- At high effort, add a no-tidying line to stop unrequested refactors. +- Ground progress claims: require each claim to match a tool result from the + session. +- State boundaries: when the user describes a problem, report findings and + stop. Do not apply fixes until asked. +- Delegate to sub-agents and let them run asynchronously. +- Give a memory file for learnings across sessions. +- Autonomous pipelines: add a "do not ask permission for reversible actions" + reminder. +- Avoid showing remaining-token counts. Deep sessions can trigger context + anxiety. +- Long sessions: ask for a final summary written for a reader who did not + watch the work. No arrow chains or made-up labels. + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| Fable 5.1 | `claude-fable-5-1` | Current | +| Fable 5 | `claude-fable-5` | Active | +| Mythos 5.1 | `claude-mythos-5-1` | Project Glasswing only | diff --git a/references/models/claude-haiku-5-5.md b/references/models/claude-haiku-5-5.md new file mode 100644 index 0000000..6723015 --- /dev/null +++ b/references/models/claude-haiku-5-5.md @@ -0,0 +1,81 @@ +# Migrating to Claude Haiku 5.5: breaking changes and fixes + +What breaks when you move code from Haiku 4.5 to `claude-haiku-5-5`, the fix for each, and what to re-tune. + +Model ID: `claude-haiku-5-5` (Amazon Bedrock: `anthropic.claude-haiku-5-5`) +Pricing: $0.10 / $0.50 per million input / output tokens for prompts up to +100K tokens. $0.50 / $2.50 above 100K tokens. (Haiku 4.5: $1 / $5 at any +length.) +Context / output: 1M tokens / 128K tokens (Haiku 4.5: 200K / 64K) +Last verified: 2026-10-11 +Source: Anthropic API migration guide (bundled `claude-api` skill, cached +2026-10-06) and platform docs + +This is the current Haiku, for high-volume and latency-sensitive work: +classification, routing, extraction, and sub-agent tasks. + +## Changelog (vs Haiku 4.5) + +- First Haiku with `effort` levels (`low` to `max`, default `medium`). +- Adaptive thinking on by default. Manual budgets removed. +- Sampling parameters and assistant prefill removed. +- Newer tokenizer: about 30% more tokens for the same text. +- Safety classifiers can decline requests. No server-side fallback exists. +- Larger limits, browser use tool, mid-conversation system messages, + per-message effort, compaction, programmatic tool calling, tool search. + +## Required changes (will 400) + +| Change | Detail | +| :----- | :----- | +| Manual thinking budget | `{type: "enabled", budget_tokens: N}` 400s. Use `{type: "adaptive"}` (or omit) plus `output_config.effort`. | +| Sampling parameters | `temperature` must be `1`, `top_p` must be `0.99`, any `top_k` 400s, and sending both `temperature` and `top_p` 400s. Omit all three. A `temperature=0` classifier: remove it and constrain output with structured outputs or an enum tool. | +| Assistant prefill | A final assistant turn 400s, even with thinking off. End `messages` with a user turn. Use `output_config.format`, an enum tool, or a prompt instruction. | +| Computer use | `computer_20250124` 400s on the Claude API and Google Cloud. Use `computer_toolset_20260801`. Replace the `fine-grained-tool-streaming-2025-05-14` header with `eager_input_streaming: true`. Do not assume the Bedrock exception that Opus 5.5 and Sonnet 5.5 have. | +| Preserved thinking | Replaying a thinking block after editing `system`, `tools`, or earlier `messages` 400s on accounts created on or after 2026-08-31. Keep histories append-only. | +| Refusals | Four categories: `cyber`, `bio`, `frontier_llm`, `general_harms`. Check `stop_reason` first. Do **not** send `fallbacks`: `"default"` leaves a decline declined, and a model list 400s. Handle it in the client. | + +## Silent changes + +| Change | Action | +| :----- | :----- | +| Thinking on by default | Read content blocks by `type`, never `content[0]`. A response can start with `thinking` blocks. | +| Thinking counts toward `max_tokens` | A cap sized for a one-word answer (for example 50) can be consumed by thinking. Raise it or lower effort. | +| Thinking text omitted | Set `display: "summarized"` to see reasoning. Keep empty `thinking` blocks. | +| Tokenizer | Recount with `count_tokens` against `claude-haiku-5-5`. Re-baseline `max_tokens` and cost. | +| Disabled thinking | Still accepted at `low`, `medium`, `high`. 400s at `xhigh` and `max`. Prefer adaptive at `low`. | +| Forced `tool_choice` | Still accepted, but the response skips thinking. Use `auto` plus a prompt when the call needs thought. | +| Cache minimum | 512 tokens (was 4,096). Short prompts now create cache entries. | +| Thinking blocks | Stay with the account that produced them. Replay a stored conversation through the same account. | + +## Effort guide + +| Level | Use | +| :---- | :-- | +| `low` | Chat, short tool tasks, simple high-volume routes. Cheapest and fastest. | +| `medium` (default) | Most work, including agentic coding. | +| `high` | Knowledge work, long agent tasks, strict instruction following. | +| `xhigh`, `max` | Only with a measured gain. Compare against Sonnet 5.5 first. | + +To get less thinking, lower effort. A "answer directly" prompt barely helps. + +## Prompt and behavior tuning (optional) + +- Search tools: put today's date in the system prompt or tool description. For + long prompts or `low` effort, add a line saying which facts may have changed + since training. Avoid blanket "always search" rules. +- JSON output plus the caller's own tools with thinking off can skip a needed + tool call. Keep thinking on, or add: "The JSON output format applies to your + final answer only. When you need a tool, call it first." +- Check for empty replies at `xhigh` in multi-turn chat. +- Long agent prompts at `low` effort can stop early. Add a keep-working + instruction, or raise effort. +- Add a verification instruction for coding agents at `low` and `medium`. +- Deliver mid-turn user input as a text block after the last `tool_result`. + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| Haiku 5.5 | `claude-haiku-5-5` | Current | +| Haiku 4.5 | `claude-haiku-4-5` | Active | diff --git a/references/models/claude-opus-4-6.md b/references/models/claude-opus-4-6.md index 44e16ff..fc0410f 100644 --- a/references/models/claude-opus-4-6.md +++ b/references/models/claude-opus-4-6.md @@ -3,6 +3,8 @@ Released: February 5, 2026 Model ID: `claude-opus-4-6` Pricing: $5 / $25 per million input / output tokens +Status: legacy. Superseded by `claude-opus-4-8.md` and `claude-opus-5-5.md`. +Last verified: legacy, written at the Opus 4.6 launch (February 2026) ## Key features diff --git a/references/models/claude-opus-4-8.md b/references/models/claude-opus-4-8.md index 86ce8bb..1e787d6 100644 --- a/references/models/claude-opus-4-8.md +++ b/references/models/claude-opus-4-8.md @@ -3,6 +3,9 @@ Released: May 28, 2026 Model ID: `claude-opus-4-8` Pricing: $5 / $25 per million input / output tokens (unchanged from 4.7) +Status: legacy. Superseded by `claude-opus-5-5.md`. Still the required layer +when upgrading from Opus 4.7 or older. +Last verified: legacy, written at the Opus 4.8 launch (May 2026) ## Changelog (vs Opus 4.7) diff --git a/references/models/claude-opus-5-5.md b/references/models/claude-opus-5-5.md new file mode 100644 index 0000000..933dee8 --- /dev/null +++ b/references/models/claude-opus-5-5.md @@ -0,0 +1,98 @@ +# Migrating to Claude Opus 5.5: breaking changes and fixes + +What breaks when you move code to `claude-opus-5-5`, the fix for each, and what to re-tune. + +Model ID: `claude-opus-5-5` (Amazon Bedrock: `anthropic.claude-opus-5-5`) +Pricing: $4 / $20 per million input / output tokens (Opus 5: $5 / $25) +Cache reads: $0.20 per million tokens +Context / output: 1M tokens / 128K tokens +Last verified: 2026-10-11 +Source: Anthropic API migration guide (bundled `claude-api` skill, cached +2026-10-06) and platform docs + +This is the current Opus and the default upgrade target for Claude code. +Layered guide: coming from Opus 4.8 or older, apply the earlier sections +first (see "Upgrade paths"). + +## Changelog (vs Opus 5) + +- **Lower price**: 20% cheaper per token, 60% cheaper for cache reads. +- **Thinking always on**: `thinking: {type: "disabled"}` is gone. +- **Effort default is `medium`** (Opus 5 defaulted to `high`). +- **Forced `tool_choice` removed**. +- **Preserved thinking**: thinking blocks are tied to the model and to an + unedited conversation prefix. +- **Computer use via toolset only** on the Claude API and Google Cloud. +- **Progress notes between tool calls** come back as `thinking` blocks. +- **New `bio` safety category** joins `cyber` and `reasoning_extraction`. + +## Required changes (will 400) + +| Change | Detail | +| :----- | :----- | +| Disabled thinking | `{type: "disabled"}` and `{type: "enabled", budget_tokens}` both 400 at every effort level. Omit `thinking` or send `{type: "adaptive"}`. Use `effort: "low"` to cut latency. | +| Forced tool use | `tool_choice` `any` and `tool` 400 (also on `count_tokens` and Batches). Use `auto` + `strict: true` + a prompt instruction, then check a call happened. Use `output_config.format` when the forced call only existed to get JSON. | +| Computer use | `computer_20251124` 400s on the Claude API and Google Cloud. Use `{"type": "computer_toolset_20260801"}` (no `name`, no display size, no beta header). Amazon Bedrock still accepts `computer_20251124`. The agent loop changes: the action is the `tool_use` block `name`, several calls per turn, echo `toolset_name` on every result. | +| Preserved thinking | A replayed thinking block after an edit to `system`, `tools`, or earlier `messages` returns 400 on accounts created on or after 2026-08-31. Older accounts opt in via `thinking.block_binding.prefix_mismatch_behavior`. Keep histories append-only. | +| Refusals | Check `stop_reason == "refusal"` before reading `content`. Ship a fallback opt-in. `reasoning_extraction` is not retried on a fallback. | + +## Silent changes (no error, behavior differs) + +| Change | Action | +| :----- | :----- | +| Default effort `medium` | Set `output_config.effort` explicitly. Re-run the effort sweep. `medium` here often matches Opus 5 at `high`. | +| More thinking per level | At `xhigh` and `max`, expect longer turns and more output tokens. Lower effort before adding "think less" prompts. | +| `thinking.display` default `"omitted"` | Set `"summarized"` to show reasoning. Keep empty `thinking` blocks when replaying. | +| Progress updates as `thinking` | Set `display: "updates"` (beta `thinking-display-updates-2026-08-18`) or render non-empty `thinking` blocks ahead of the `tool_use`. | +| `max_tokens` | Size for thinking plus reply. 64K is a good start for long agentic coding. | +| Rate limits | Separate pool from Opus 5. Re-check the tier. | +| Priority Tier | Not supported. | +| Fast mode | Claude API only, $8 / $40 per million tokens. | + +## Upgrade paths + +| From | Apply, in order | +| :--- | :-------------- | +| Opus 5 | This guide only. | +| Opus 4.8 | `claude-opus-4-8.md` (re-tune), Opus 5 changes (thinking on by default, prompt re-tuning), then this guide. | +| Opus 4.7 / 4.6 or older | `claude-opus-4-8.md` (includes the 4.7 breaking changes), Opus 5 changes, then this guide. | +| Sonnet 5 / 5.5 | Request surface already matches. Re-baseline for Opus pricing and rate limits. | + +Opus 5 specifics to carry through on a 4.x upgrade: thinking is on when +`thinking` is omitted, the cache minimum is 512 tokens, the 1M context window +is the default, and Opus 5 writes longer visible output than 4.8 (add +conciseness and scope-discipline instructions). + +## Prompt and behavior tuning (optional) + +- Remove instructions that stood in for thinking. A prompt that asks the model + to write its reasoning into the response can be declined with + `reasoning_extraction`. Read `display: "summarized"` blocks instead. +- Delete "don't think" rules. The model cannot comply and tag leakage rises. +- Re-test visual scaffolding. Chart and screenshot reading improved at every + effort level without extra tooling. +- Re-test Opus 5 verbosity, verification, and scope instructions. Keep them + as a starting point, then A/B each. +- For frontend work, name specific default patterns to avoid. "No generic AI + look" mostly swaps one default for another. +- To vary effort without losing the prompt cache, use per-message effort + (beta `mid-conversation-output-config-2026-07-01`). Changing top-level + `effort` invalidates the cache. + +## Validation notes + +- Smoke test: assert `response.model.startswith("claude-opus-5-5")`. +- Prefix collision: a check for `claude-opus-5` also matches + `claude-opus-5-5`. Add a `not startswith("claude-opus-5-5")` guard. +- Test a long tool loop with thinking blocks replayed unchanged. +- Test a forced-fallback refusal path. + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| Opus 5.5 | `claude-opus-5-5` | Current | +| Opus 5 | `claude-opus-5` | Active | +| Opus 4.8 | `claude-opus-4-8` | Active | +| Opus 4.7 | `claude-opus-4-7` | Active | +| Opus 4.6 | `claude-opus-4-6` | Active | diff --git a/references/models/claude-sonnet-4-5.md b/references/models/claude-sonnet-4-5.md index 0cc35ee..120ec32 100644 --- a/references/models/claude-sonnet-4-5.md +++ b/references/models/claude-sonnet-4-5.md @@ -3,6 +3,8 @@ Released: September 29, 2025 Model ID: `claude-sonnet-4-5` Pricing: $3 / $15 per million input / output tokens +Status: legacy. Superseded by `claude-sonnet-5-5.md`. +Last verified: legacy, written at the Sonnet 4.5 launch (September 2025) ## Key features diff --git a/references/models/claude-sonnet-5-5.md b/references/models/claude-sonnet-5-5.md new file mode 100644 index 0000000..b733f7c --- /dev/null +++ b/references/models/claude-sonnet-5-5.md @@ -0,0 +1,90 @@ +# Migrating to Claude Sonnet 5.5: breaking changes and fixes + +What breaks when you move code to `claude-sonnet-5-5`, the fix for each, and what to re-tune. + +Model ID: `claude-sonnet-5-5` (Amazon Bedrock: `anthropic.claude-sonnet-5-5`) +Pricing: $2 / $10 per million input / output tokens (same as Sonnet 5) +Cache reads: $0.20 per million tokens +Context / output: 1M tokens / 128K tokens +Last verified: 2026-10-11 +Source: Anthropic API migration guide (bundled `claude-api` skill, cached +2026-10-06) and platform docs + +This is the current Sonnet. Layered guide: coming from Sonnet 4.6 or older, +apply the earlier sections first (see "Upgrade paths"). + +## Changelog (vs Sonnet 5) + +- **`disabled` thinking removed**: use `{type: "between_tools"}` as the + lowest thinking setting. +- **Forced `tool_choice` removed**. +- **Preserved thinking**: thinking blocks are tied to the model and to an + unedited conversation prefix. +- **Computer use via toolset only** on the Claude API and Google Cloud. +- **Advisor tool** accepts fewer advisors. +- **Effort levels recalibrated**. Default is still `high`. +- **New**: mid-conversation system messages, tool changes, per-message effort, + task budgets, 512-token cache minimum. + +## Required changes (will 400) + +| Change | Detail | +| :----- | :----- | +| Disabled thinking | `{type: "disabled"}` 400s. First try adaptive thinking at `effort: "low"`. If the route must stay thinking-off, send `{type: "between_tools"}` at effort `high` or below. | +| `between_tools` limits | No other field inside `thinking`. No per-message effort change. Accepted by Sonnet 5.5 only, so drop it from any body re-sent to another model. | +| Forced tool use | `tool_choice` `any` and `tool` 400 (also on `count_tokens`). Use `auto` + `strict: true` + a prompt instruction, or structured outputs. | +| Computer use | `computer_20251124` 400s on the Claude API and Google Cloud. Use `computer_toolset_20260801`. Amazon Bedrock still accepts `computer_20251124`. Do not prune old screenshots client-side: it invalidates later thinking blocks. | +| Preserved thinking | Enforced by default on the Claude API and Amazon Bedrock for accounts created on or after 2026-08-31. `block_binding` works only with adaptive thinking. Keep histories append-only. | +| Advisor tool | Rejects Opus 4.8, 4.7, 4.6, Sonnet 5, and Sonnet 4.6 advisors. Accepted advisors return encrypted advice. | +| Refusals | Five `stop_details` categories: `cyber`, `bio`, `frontier_llm`, `reasoning_extraction`, `general_harms`. Server-side fallback retries only `cyber` and `frontier_llm`. | + +## Silent changes + +| Change | Action | +| :----- | :----- | +| Effort levels recalibrated | Set `effort` explicitly. Start at `medium` for agentic coding and multistep tool use. Start at `low` for chat, extraction, classification, and search. | +| Thinking before most replies | From `medium` up, the model thinks briefly before almost every reply. Lower effort to cut time to first token. A "think less" prompt barely helps. | +| Progress updates as `thinking` | Set `display: "updates"` (beta) with adaptive thinking. With `between_tools`, the notes return as text and no `display` is allowed. | +| Same tokenizer as Sonnet 5 | Token counts unchanged from Sonnet 5. | +| Rate limits | Own pool. Priority Tier not supported. | + +## Upgrade paths + +| From | Apply, in order | +| :--- | :-------------- | +| Sonnet 5 | This guide only. | +| Sonnet 4.6 | Sonnet 5 changes (adaptive thinking on by default, sampling params rejected, about 30% more tokens, `display` default `"omitted"`), then this guide. Skip the Sonnet 5 effort advice. | +| Sonnet 4.5 or older, Haiku 4.5 | Sonnet 4.6 changes first (prefill removed, explicit effort, beta headers), then Sonnet 5, then this guide. | + +## Prompt and behavior tuning (optional) + +- Remove workarounds for what improved: refusal steering, tool-call retry + shims, "do not be lazy". +- Remove tool-discouraging language ("minimize tool calls"). The model follows + it literally and may skip connected tools. +- Deliver mid-turn user input as a text block after the last `tool_result`. + Never put user text inside a `tool_result`. A mid-task user message placed + right after a tool result can be read as prompt injection. +- Skip task budgets on interactive sessions. The countdown arrives as a system + message after every tool result and can trigger the same reaction. +- At `low` effort, coding agents may report a change done without running a + check. Add a "run a real check before reporting done" instruction. +- Accept near-miss tool names (case differences) or return an `is_error` + result that states the exact expected name. +- Give crop, zoom, or code tools for dense charts and technical drawings. + +## Validation notes + +- Smoke test: assert `response.model.startswith("claude-sonnet-5-5")`. +- Prefix collision: a check for `claude-sonnet-5` also matches + `claude-sonnet-5-5`. Add a `not startswith("claude-sonnet-5-5")` guard. +- Test the thinking-off route at `low` effort first, then `between_tools`. + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| Sonnet 5.5 | `claude-sonnet-5-5` | Current | +| Sonnet 5 | `claude-sonnet-5` | Active | +| Sonnet 4.6 | `claude-sonnet-4-6` | Active | +| Sonnet 4.5 | `claude-sonnet-4-5` | Active | diff --git a/references/models/gemini-3-8-flash.md b/references/models/gemini-3-8-flash.md new file mode 100644 index 0000000..86de8e5 --- /dev/null +++ b/references/models/gemini-3-8-flash.md @@ -0,0 +1,79 @@ +# Migrating to Gemini 3.8 Flash: breaking changes and fixes + +What changes when you move code to `gemini-3.8-flash`, the fix for each, and what to re-tune. + +Model ID: `gemini-3.8-flash` (generally available) +Context / output: 1M tokens / 64K tokens +Default thinking level: `medium` +Pricing: $0.75 / $3.75 per million input / output tokens through Dec 31, 2026. +$1.50 / $7.50 from Jan 1, 2027. The introductory rate also applies to Gemini +3.6 Flash. +Last verified: 2026-10-11 +Source: Google "What's new in Gemini 3.8 Flash" (last updated 2026-10-09) and +the Gemini API pricing page. The release date is not stated on either page. + +Google's most intelligent Flash model. The pricing page lists no Pro model +newer than `gemini-3.1-pro-preview`. Gemini 3.6 Flash stays supported. + +## Current model lineup (pricing page, per 1M tokens in / out) + +| Model | Model ID | Price | +| :---- | :------- | :---- | +| Gemini 3.8 Flash | `gemini-3.8-flash` | $0.75 / $3.75 | +| Gemini 3.6 Flash | `gemini-3.6-flash` | $0.75 / $3.75 | +| Gemini 3.5 Flash-Lite | `gemini-3.5-flash-lite` | $0.30 / $2.50 | +| Gemini 3.1 Flash-Lite | `gemini-3.1-flash-lite` | $0.25 / $1.50 | +| Gemini 3.1 Pro Preview | `gemini-3.1-pro-preview` | $2.00 / $12.00 (up to 200K) | + +## Thinking levels + +| Level | Use | +| :---- | :-- | +| `low` | Latency-critical work: incident response, real-time chat, drafts, fast data analysis | +| `medium` (default) | Complex code and agentic use cases | +| `high` | Deep reasoning, mathematics, hard multi-step tasks | + +`minimal` returns an error on 3.8 Flash. The model uses smaller reasoning +steps, calls tools iteratively, and verifies its work, so it can spend more +tokens on long tasks. Lower the level on everyday tasks to cut token use. + +## Migration checklist + +| Step | Change | +| :--- | :----- | +| 1 | Set the model string to `gemini-3.8-flash`. | +| 2 | Replace `thinking_budget` with the string enum `thinking_level`. Sending both returns 400. | +| 3 | Remove `temperature`, `top_p`, `top_k`, `candidate_count`. `candidate_count` is unsupported in Gemini 3 and later. Secondary guides disagree on whether the sampling parameters error or are silently ignored. Remove them and test. | +| 4 | Remove prefilled model turns. Use server-side `previous_interaction_id` for multi-turn conversations. | +| 5 | Function calling: put multimodal assets inside the response payload, format inline instructions with `\n\n`, and for `generateContent` make every `FunctionResponse` include `call_id` and `name`. For `Malformed_Function_Call` errors tied to text before a tool call, see the linked workarounds on the Google page. | +| 6 | Preserve thought signatures per the Gemini 3 requirements. Thought preservation across turns is on by default. | + +## Behavior notes + +- Default effort fell from high to medium across the 3.5 generation. A route + that relied on the old default now thinks less. Set `thinking_level` + explicitly and re-run evals. +- If the model over-calls tools, lower the thinking level. +- The Antigravity agent in Managed Agents and the Antigravity SDK default to + Gemini 3.8 Flash. Override via `agent_config`. +- The older Gemini 3 developer guide is deprecated. Prefer the + "What's new in Gemini 3.8 Flash" page. +- `gemini-2.5-flash-image` shuts down 2026-10-02. + +## Upgrade paths + +| From | Notes | +| :--- | :---- | +| Gemini 3.6 / 3.7 Flash | Mostly a model-ID swap. Apply steps 2 to 6 if not done, and drop `minimal`. | +| Gemini 3.5 Flash | Apply the full checklist. Re-check the default thinking level. | +| Gemini 2.5, 3 Preview | Apply the full checklist. Move from `thinking_budget` to `thinking_level` and re-validate function calling. | +| Another provider | Follow `provider-checklists.md` (cross-provider section), then this guide. | + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| Gemini 3.8 Flash | `gemini-3.8-flash` | Current (GA) | +| Gemini 3.6 Flash | `gemini-3.6-flash` | Supported | +| Gemini 3.1 Pro Preview | `gemini-3.1-pro-preview` | Preview | +| Gemini 2.5 Pro | `gemini-2.5-pro` | Older | diff --git a/references/models/gpt-5-5.md b/references/models/gpt-5-5.md index e5dc516..2d942fe 100644 --- a/references/models/gpt-5-5.md +++ b/references/models/gpt-5-5.md @@ -5,6 +5,9 @@ Model ID: `gpt-5.5` Pricing: $5 / $30 per million input / output tokens Cached input: $0.50 per million tokens Long-context (>270k input): 2x input, 1.5x output +Status: legacy. Superseded by `gpt-6.md`. Still the required layer when +upgrading from GPT-4o or o-series. +Last verified: legacy, written at the GPT-5.5 launch (April to May 2026) ## Variants diff --git a/references/models/gpt-6.md b/references/models/gpt-6.md new file mode 100644 index 0000000..cf64e07 --- /dev/null +++ b/references/models/gpt-6.md @@ -0,0 +1,102 @@ +# Migrating to GPT-6 (Astra, 6.1 Sol, Sol, Luna): breaking changes and fixes + +What changes when you move code from GPT-5.x to the GPT-6 family, the fix for each, and what to re-tune. + +Model IDs: `gpt-6-astra`, `gpt-6.1-sol`, `gpt-6-sol`, `gpt-6-luna` +Context / output: 1,050,000 tokens (922,000 max input) / 128,000 tokens +Knowledge cutoff: Apr 30, 2026 (Astra and 6.1 Sol) +Last verified: 2026-10-11 +Source: OpenAI "Using GPT-6" guide and the `gpt-6-astra` and `gpt-6.1-sol` +model pages. Sol and Luna prices and all release dates come from secondary +sources and are marked below. Re-check the OpenAI model pages before quoting. + +## Variants + +| Variant | Model ID | Use case | Price in / out per 1M tokens | +| :------ | :------- | :------- | :--------------------------- | +| GPT-6 Astra | `gpt-6-astra` | Highest intelligence: demanding reasoning, coding, professional work | $10 / $50 (cached $1, cache writes $12.50) | +| GPT-6.1 Sol | `gpt-6.1-sol` | Balanced speed, cost, intelligence | $2 / $10 (cached $0.10, cache writes $2.50) | +| GPT-6 Sol | `gpt-6-sol` | Existing model. Review the migration notes before moving to 6.1 Sol | $2 / $10 (secondary source) | +| GPT-6 Luna | `gpt-6-luna` | Fastest, cheapest, high-volume focused tasks | $0.10 / $0.50 (secondary source) | + +Prompts over 272K input tokens bill the whole request at 2x input and cache +rates and 1.5x output. Fast mode is 2x, Ultrafast is 6x, Batch and Flex are +50% lower, regional processing adds 10% (6.1 Sol page). + +Secondary-source release dates (unverified): Astra 2026-09-03, GPT-6 Sol and +Luna 2026-09-22, GPT-6.1 Sol 2026-09-29 or 2026-09-30. + +## Reasoning effort + +Set `reasoning.effort` (Responses API) or `reasoning_effort` (Chat +Completions). + +| Model | Levels | Notes | +| :---- | :----- | :---- | +| GPT-6.1 Sol | `low`, `medium` (default), `high`, `xhigh`, `max` | `none` and `minimal` unsupported | +| GPT-6 Astra | `low`, `medium`, `high`, `xhigh`, `max` | `none` unsupported. Use `low` instead | +| GPT-6 Sol, GPT-6 Luna | Support `none` | | + +## Required changes + +| Area | Change | +| :--- | :----- | +| Endpoint | Use the **Responses API** for tool calling. Astra and 6.1 Sol support Chat Completions only for requests without tools. Sol and Luna support function calling in Chat Completions only with `reasoning_effort: "none"`. | +| Sampling | When effort is not `none`, remove `temperature`, `top_p`, `top_logprobs`. Chat Completions: also remove `logprobs`. Responses: remove `message.output_text.logprobs` from `include`. | +| `minimal` effort | Replace with `low`. Compare on representative tasks. | +| `none` effort (Astra, 6.1 Sol) | Replace with `low`. | +| Prompt caching | Replace `prompt_cache_retention` with `prompt_cache_options.ttl` set to `"30m"` when migrating from GPT-5.5 or earlier. Review cache boundary and cache-write billing changes. | +| Model ID | Set `model` to the target ID. Astra and 6.1 Sol list one snapshot each: the bare ID. | + +OpenAI suggests Codex with the OpenAI Docs skill: +`$openai-docs migrate this project to the GPT-6 model family`. + +## New API features + +- **Async tool calling**: set `async: true` on a function or custom tool, + then return the result later with the original `call_id`. +- **Mid-turn steering**: send user instructions while the model works. +- **`configuration_update` input item**: change reasoning effort mid- + conversation without altering the prompt prefix, so caching survives. +- **Fast and Ultrafast modes**. Data residency differs: Astra Fast mode has + no EU residency and no latency SLA. +- Carried over from GPT-5.6: computer use, Structured Outputs, streaming, + Programmatic Tool Calling, multi-agent orchestration, prompt caching, + persisted reasoning, compaction, pro mode. + +## Prompt and behavior tuning (optional) + +- **Initiative**: Astra asks clarifying questions more than earlier models. + Say to act when the request implies authorization, and to ask for approval + once a concrete result is ready. +- **Instruction priority**: Astra weighs `AGENTS.md` and skill files heavily. + Audit them. State that user instructions override skill guidance. +- **Style**: Astra favors long, formatted answers with lists and tables. State + the style you want if you need prose. +- **Subagents**: Astra may delegate less than you want. Say when and how. +- **Testing**: Astra tests thoroughly, sometimes beyond what a small change + needs. Calibrate the scope. +- Carry-over from GPT-5.5 guidance: outcome-first prompts, drop legacy + scaffolding, keep few-shot examples for format calibration. + +## Upgrade paths + +| From | Notes | +| :--- | :---- | +| GPT-5.5 | Apply the required changes above. Re-baseline effort: Astra defaults are not documented, 6.1 Sol defaults to `medium`. See `gpt-5-5.md` for the earlier surface. | +| GPT-5.x, GPT-4o, o-series | Apply the `gpt-5-5.md` steps first (Responses API, effort, structured outputs), then this guide. | +| GPT-6 Sol | Review the migration notes before moving to 6.1 Sol. | + +Search results also mention a GPT-5.6 model page. Treat GPT-5.6 as an +intermediate release and verify its status on the OpenAI models page before +targeting it. + +## Model ID reference + +| Version | Model ID | Status | +| :------ | :------- | :----- | +| GPT-6 Astra | `gpt-6-astra` | Current | +| GPT-6.1 Sol | `gpt-6.1-sol` | Current | +| GPT-6 Sol | `gpt-6-sol` | Existing | +| GPT-6 Luna | `gpt-6-luna` | Current | +| GPT-5.5 | `gpt-5.5` | Previous generation (see `gpt-5-5.md`) | diff --git a/references/provider-checklists.md b/references/provider-checklists.md index 322b94c..08114b2 100644 --- a/references/provider-checklists.md +++ b/references/provider-checklists.md @@ -11,10 +11,19 @@ Provider-specific reminders to use during migration. - Verify structured output mode and schema handling. - Run at least one streamed and one non-streamed smoke test where relevant. -For version-specific changelogs, breaking changes, and quirks, see the -dedicated model guides in `references/models/` (e.g. `claude-opus-4-8.md`). +- Check for the 5.x-era breaking changes when targeting the latest models: + `thinking: {type: "disabled"}` and `budget_tokens` rejected, forced + `tool_choice` (`any`, `tool`) rejected, sampling parameters and assistant + prefill rejected, computer use via `computer_toolset_20260801`. +- Handle `stop_reason: "refusal"` before reading `content`. Add a fallback. +- Keep conversations append-only when replaying thinking blocks (preserved + thinking). +- Set `output_config.effort` explicitly. Defaults differ per model. -Reference: [Anthropic model deprecations](https://docs.anthropic.com/en/docs/resources/model-deprecations) +For version-specific changelogs, breaking changes, and quirks, start at +`references/models/INDEX.md` (e.g. `claude-opus-5-5.md`). + +Reference: [Anthropic model deprecations](https://platform.claude.com/docs/en/about-claude/model-deprecations) ## OpenAI-family migration checklist @@ -23,11 +32,15 @@ Reference: [Anthropic model deprecations](https://docs.anthropic.com/en/docs/res - Confirm any reasoning-effort/config fields are valid for target model. - Validate JSON mode/structured output behavior if used. - Re-check system prompt behavior under new model family. +- Use the Responses API for tool calling on the newest models. +- Remove `temperature`, `top_p`, and `logprobs` when reasoning is on. +- Replace unsupported effort values (`none`, `minimal`) per the target model. +- Re-check prompt caching parameters (for example `prompt_cache_options.ttl`). For version-specific changelogs, reasoning effort levels, and migration steps, -see the dedicated model guides in `references/models/` (e.g. `gpt-5-5.md`). +start at `references/models/INDEX.md` (e.g. `gpt-6.md`). -Reference: [OpenAI model endpoint compatibility](https://platform.openai.com/docs/models) +Reference: [OpenAI models](https://developers.openai.com/api/docs/models) ## Gemini-family migration checklist @@ -36,6 +49,13 @@ Reference: [OpenAI model endpoint compatibility](https://platform.openai.com/doc - Re-check safety settings and blocked-output behavior. - Validate multimodal pathways if image/file inputs are used. - Confirm output format consistency against existing parsers. +- Replace `thinking_budget` with `thinking_level`. Do not send both. +- Remove `temperature`, `top_p`, `top_k`, `candidate_count`, and prefilled + model turns on Gemini 3 and later. +- Re-check function-calling response shape and thought-signature handling. + +For version-specific steps, start at `references/models/INDEX.md` (e.g. +`gemini-3-8-flash.md`). Reference: [Gemini model versions](https://ai.google.dev/gemini-api/docs/models) diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py new file mode 100644 index 0000000..c85dec1 --- /dev/null +++ b/scripts/check_freshness.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Report model guides whose `Last verified:` date is too old. + +Exit code 0 when every dated guide is fresh, 1 when any is stale. Legacy guides +(`Last verified: legacy ...`) are skipped. Used by the weekly freshness +workflow, which opens an issue with this report. + +Usage: python3 scripts/check_freshness.py [root] [--max-age-days 30] [--today YYYY-MM-DD] +""" + +from __future__ import annotations + +import argparse +import re +import sys +from datetime import date +from pathlib import Path + + +def stale_guides(root: Path, max_age_days: int, today: date) -> list[tuple[str, int, str]]: + """Return (guide, age_in_days, source_line) for every dated guide past max age.""" + stale = [] + for guide in sorted((root / "references" / "models").glob("*.md")): + if guide.name == "INDEX.md": + continue + head = "\n".join(guide.read_text(encoding="utf-8").splitlines()[:20]) + verified = re.search(r"^Last verified: (\d{4})-(\d{2})-(\d{2})\b", head, re.MULTILINE) + if not verified: + continue # legacy guide (or validate_skill.py will flag it) + age = (today - date(*map(int, verified.groups()))).days + if age > max_age_days: + # Source may wrap onto continuation lines: join until a blank line or the next key. + source = re.search( + r"^Source: (.+(?:\n(?![A-Za-z ]+: |\n|#).+)*)", head, re.MULTILINE + ) + stale.append((guide.name, age, " ".join(source.group(1).split()) if source else "")) + return stale + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("root", nargs="?", default=".") + ap.add_argument("--max-age-days", type=int, default=30) + ap.add_argument("--today", default=None, help="override today's date (YYYY-MM-DD)") + args = ap.parse_args() + + today = date.fromisoformat(args.today) if args.today else date.today() + stale = stale_guides(Path(args.root).resolve(), args.max_age_days, today) + if not stale: + print(f"All dated model guides were verified within {args.max_age_days} days.") + return + + print(f"{len(stale)} model guide(s) older than {args.max_age_days} days:\n") + for name, age, source in stale: + print(f"- references/models/{name}: {age} days old. Source: {source}") + print( + "\nRe-check each guide against the provider's official page, fix any drift, " + "then update its `Last verified:` line. Also check whether the provider " + "shipped a model that has no guide in references/models/INDEX.md." + ) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 0bbbc2f..56b60a9 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -18,6 +18,7 @@ "references/provider-checklists.md", "references/validation-proof.md", "references/benchmarking.md", + "references/breaking-changes.md", "references/output-style.md", "templates/migration-starter-prompt.md", "examples/example-prompts.md", @@ -102,11 +103,49 @@ def validate_skill(root: Path) -> None: fail(f"Example output missing section: {heading}") +def validate_model_guides(root: Path) -> None: + """Every model guide needs metadata and an INDEX.md entry.""" + models_dir = root / "references" / "models" + index_path = models_dir / "INDEX.md" + if not index_path.is_file(): + fail("Missing required file: references/models/INDEX.md") + index = index_path.read_text(encoding="utf-8") + + for guide in sorted(models_dir.glob("*.md")): + if guide.name == "INDEX.md": + continue + text = guide.read_text(encoding="utf-8") + head = "\n".join(text.splitlines()[:20]) + + if not re.search(r"^Model IDs?: `", head, re.MULTILINE): + fail(f"{guide.name}: missing 'Model ID:' line in the header") + + verified = re.search(r"^Last verified: (.+)$", head, re.MULTILINE) + if not verified: + fail(f"{guide.name}: missing 'Last verified:' line in the header") + value = verified.group(1).strip() + if not (re.match(r"\d{4}-\d{2}-\d{2}\b", value) or value.startswith("legacy")): + fail( + f"{guide.name}: 'Last verified:' must start with an ISO date " + "(YYYY-MM-DD) or 'legacy'" + ) + + is_legacy = value.startswith("legacy") + if not is_legacy and not re.search(r"^Source: ", head, re.MULTILINE): + fail(f"{guide.name}: current guide needs a 'Source:' line") + if is_legacy and not re.search(r"^Status: legacy", head, re.MULTILINE): + fail(f"{guide.name}: legacy guide needs a 'Status: legacy' line") + + if guide.name not in index: + fail(f"{guide.name}: not listed in references/models/INDEX.md") + + def main() -> None: root = Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve() if not root.is_dir(): fail(f"Not a directory: {root}") validate_skill(root) + validate_model_guides(root) print("Skill package is valid.")