diff --git a/cmd/cc-session/benchmark.go b/cmd/cc-session/benchmark.go index 7875d27..9ccb045 100644 --- a/cmd/cc-session/benchmark.go +++ b/cmd/cc-session/benchmark.go @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S days := fs.Int("days", 30, "how far back to scan") minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB") maxN := fs.Int("n", 10, "max sessions to include") - model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, or fable (fable-5-1)") + model := fs.String("model", "opus", "model: opus (opus-5-5), opus-4-6, opus-4-7, opus-4-8, sonnet (sonnet-5-5), sonnet-4-6, or fable (fable-5-1)") overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session") isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)") if err := fs.Parse(reorderArgs(args)); err != nil { diff --git a/cmd/cc-session/benchmark_test.go b/cmd/cc-session/benchmark_test.go index a698660..938b61a 100644 --- a/cmd/cc-session/benchmark_test.go +++ b/cmd/cc-session/benchmark_test.go @@ -81,11 +81,12 @@ func TestResolveBenchmarkModel_GivenAcceptedModelNames_ThenReturnsPricingAndToke wantPricing bm.Pricing wantTokenCountModel string }{ - {name: "opus", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus48}, + {name: "opus", wantPricing: bm.PricingOpus55, wantTokenCountModel: bm.TokenCountModelOpus55}, {name: "opus-4-6", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus46}, {name: "opus-4-7", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus47}, {name: "opus-4-8", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus48}, - {name: "sonnet", wantPricing: bm.PricingSonnet, wantTokenCountModel: bm.TokenCountModelSonnet}, + {name: "sonnet", wantPricing: bm.PricingSonnet55, wantTokenCountModel: bm.TokenCountModelSonnet55}, + {name: "sonnet-4-6", wantPricing: bm.PricingSonnet46, wantTokenCountModel: bm.TokenCountModelSonnet46}, } for _, tt := range tests { @@ -111,7 +112,7 @@ func TestResolveBenchmarkModel_GivenUnknownModel_ThenReturnsAcceptedNames(t *tes } got := err.Error() - for _, want := range []string{"opus", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet"} { + for _, want := range []string{"opus", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet", "sonnet-4-6"} { if !strings.Contains(got, want) { t.Fatalf("error message missing accepted model %q: %s", want, got) } @@ -171,7 +172,7 @@ func TestRunBenchmark_WhenSessionHasAPIUsage_ThenUsesTokenCountingAPIForNewConte if countedText == "" { t.Fatal("countTokensFn was not called") } - wantCountModel := bm.TokenCountModelOpus48 + wantCountModel := bm.TokenCountModelOpus55 if countModel != wantCountModel { t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel) } @@ -328,7 +329,7 @@ func TestRunBenchmark_GivenSonnetModel_ThenUsesSonnetTokenCounterModel(t *testin t.Fatalf("runBenchmark returned error: %v", err) } - wantCountModel := bm.TokenCountModelSonnet + wantCountModel := bm.TokenCountModelSonnet55 if countModel != wantCountModel { t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel) } diff --git a/docs/benchmark.md b/docs/benchmark.md index 3b06af1..5d326da 100644 --- a/docs/benchmark.md +++ b/docs/benchmark.md @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data. | `--days` | 30 | How far back to scan for sessions | | `--min-kb` | 100 | Minimum JSONL file size in KB | | `--n` | 10 | Max successful session results to report | -| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `opus-5-5`, `sonnet`, or `fable` (`fable-5-1`) | +| `--model` | opus | Pricing and token-counting model: `opus` (`opus-5-5`), `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet` (`sonnet-5-5`), `sonnet-4-6`, or `fable` (`fable-5-1`) | ### Example output @@ -162,9 +162,11 @@ the historical one-shot `NewCtx × CacheWrite` behavior. `X` comes from transcript API usage and `C` comes from the Anthropic token counting API. The `--model` flag controls both pricing and the tokenizer used by -the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus 4.x -versions map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`; -`opus-5-5` maps to `claude-opus-5-5`; `sonnet` maps to `claude-sonnet-4-6`; +the token counting API. `opus` and `opus-5-5` both map to `claude-opus-5-5`; +explicit Opus 4.x versions map to `claude-opus-4-6`, `claude-opus-4-7`, or +`claude-opus-4-8`. `sonnet` and `sonnet-5-5` both map to `claude-sonnet-5-5` +($2.00/M input, $0.20/M cache read, $2.50/M cache write, $10/M output); +`sonnet-4-6` maps to `claude-sonnet-4-6`. `fable` and `fable-5-1` both map to `claude-fable-5-1`. Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates; Opus 5.5 has its own, lower rates (see the pricing table above). @@ -172,7 +174,7 @@ Fallback constants are used only for behavior that cannot be read directly from transcript usage, such as sparse tool I/O data. Cache read as a percentage of base input differs by model: Opus 4.x and Sonnet -use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers +(4.6 and 5.5) use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers are only directly comparable across rows using the same ratio. ### Simplifications diff --git a/internal/benchmark/model.go b/internal/benchmark/model.go index 50fbbbf..222cf45 100644 --- a/internal/benchmark/model.go +++ b/internal/benchmark/model.go @@ -9,8 +9,13 @@ type Pricing struct { BaseInput float64 // $/M tokens (uncached, after last breakpoint) } +// PricingOpus is the shared Opus 4.x pricing (4.6, 4.7, 4.8). var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00} -var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00} +var PricingSonnet46 = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00} + +// PricingSonnet55 is Claude Sonnet 5.5's pricing ($2 input / $10 output per +// MTok). Cache read stays at 10% of base input, like Sonnet 4.6. +var PricingSonnet55 = Pricing{CachedRead: 0.20, CacheWrite: 2.50, BaseInput: 2.00} // PricingOpus55 is Claude Opus 5.5's pricing. Its cache read is 5% of base // input rather than the 10% Opus 4.x/Sonnet use (source: Anthropic pricing @@ -26,12 +31,13 @@ var PricingOpus55 = Pricing{CachedRead: 0.20, CacheWrite: 5.00, BaseInput: 4.00} var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00} const ( - TokenCountModelOpus46 = "claude-opus-4-6" - TokenCountModelOpus47 = "claude-opus-4-7" - TokenCountModelOpus48 = "claude-opus-4-8" - TokenCountModelOpus55 = "claude-opus-5-5" - TokenCountModelSonnet = "claude-sonnet-4-6" - TokenCountModelFable = "claude-fable-5-1" + TokenCountModelOpus46 = "claude-opus-4-6" + TokenCountModelOpus47 = "claude-opus-4-7" + TokenCountModelOpus48 = "claude-opus-4-8" + TokenCountModelOpus55 = "claude-opus-5-5" + TokenCountModelSonnet46 = "claude-sonnet-4-6" + TokenCountModelSonnet55 = "claude-sonnet-5-5" + TokenCountModelFable = "claude-fable-5-1" ) // ModelConfig bundles the pricing and token-counting model for a given model alias. @@ -43,20 +49,22 @@ type ModelConfig struct { // ResolveModel maps a user-facing model alias to its ModelConfig. func ResolveModel(model string) (ModelConfig, error) { switch model { - case "sonnet": - return ModelConfig{Pricing: PricingSonnet, TokenCountModel: TokenCountModelSonnet}, nil - case "opus", "opus-4-8": + case "sonnet", "sonnet-5-5": + return ModelConfig{Pricing: PricingSonnet55, TokenCountModel: TokenCountModelSonnet55}, nil + case "sonnet-4-6": + return ModelConfig{Pricing: PricingSonnet46, TokenCountModel: TokenCountModelSonnet46}, nil + case "opus", "opus-5-5": + return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil + case "opus-4-8": return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus48}, nil case "opus-4-7": return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil case "opus-4-6": return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil - case "opus-5-5": - return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil case "fable", "fable-5-1": return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil default: - return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, fable, or fable-5-1", model) + return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, sonnet-4-6, sonnet-5-5, fable, or fable-5-1", model) } } diff --git a/internal/benchmark/model_test.go b/internal/benchmark/model_test.go index 6e68153..3fc93b1 100644 --- a/internal/benchmark/model_test.go +++ b/internal/benchmark/model_test.go @@ -15,14 +15,26 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) { { name: "sonnet", alias: "sonnet", - wantPricing: PricingSonnet, - wantTokenCount: TokenCountModelSonnet, + wantPricing: PricingSonnet55, + wantTokenCount: TokenCountModelSonnet55, + }, + { + name: "sonnet-4-6", + alias: "sonnet-4-6", + wantPricing: PricingSonnet46, + wantTokenCount: TokenCountModelSonnet46, + }, + { + name: "sonnet-5-5", + alias: "sonnet-5-5", + wantPricing: PricingSonnet55, + wantTokenCount: TokenCountModelSonnet55, }, { name: "opus", alias: "opus", - wantPricing: PricingOpus, - wantTokenCount: TokenCountModelOpus48, + wantPricing: PricingOpus55, + wantTokenCount: TokenCountModelOpus55, }, { name: "opus-4-8", diff --git a/internal/tokens/tokens.go b/internal/tokens/tokens.go index 674b4e4..065d1d7 100644 --- a/internal/tokens/tokens.go +++ b/internal/tokens/tokens.go @@ -29,7 +29,7 @@ const ( baseRetryDelay = 500 * time.Millisecond ) -const DefaultCountTokensModel = "claude-sonnet-4-6" +const DefaultCountTokensModel = "claude-sonnet-5-5" // CountTokensAPI calls the Anthropic count_tokens endpoint. // Resolves the API key from: env ANTHROPIC_API_KEY → config file path in