Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmd/cc-session/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S
days := fs.Int("days", 30, "how far back to scan")
minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB")
maxN := fs.Int("n", 10, "max sessions to include")
model := fs.String("model", "opus", "model: opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, or fable (fable-5-1)")
model := fs.String("model", "opus", "model: opus (opus-5-5), opus-4-6, opus-4-7, opus-4-8, sonnet (sonnet-5-5), sonnet-4-6, or fable (fable-5-1)")
overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session")
isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)")
if err := fs.Parse(reorderArgs(args)); err != nil {
Expand Down
11 changes: 6 additions & 5 deletions cmd/cc-session/benchmark_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,12 @@ func TestResolveBenchmarkModel_GivenAcceptedModelNames_ThenReturnsPricingAndToke
wantPricing bm.Pricing
wantTokenCountModel string
}{
{name: "opus", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus48},
{name: "opus", wantPricing: bm.PricingOpus55, wantTokenCountModel: bm.TokenCountModelOpus55},
{name: "opus-4-6", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus46},
{name: "opus-4-7", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus47},
{name: "opus-4-8", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus48},
{name: "sonnet", wantPricing: bm.PricingSonnet, wantTokenCountModel: bm.TokenCountModelSonnet},
{name: "sonnet", wantPricing: bm.PricingSonnet55, wantTokenCountModel: bm.TokenCountModelSonnet55},
{name: "sonnet-4-6", wantPricing: bm.PricingSonnet46, wantTokenCountModel: bm.TokenCountModelSonnet46},
}

for _, tt := range tests {
Expand All @@ -111,7 +112,7 @@ func TestResolveBenchmarkModel_GivenUnknownModel_ThenReturnsAcceptedNames(t *tes
}

got := err.Error()
for _, want := range []string{"opus", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet"} {
for _, want := range []string{"opus", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet", "sonnet-4-6"} {
if !strings.Contains(got, want) {
t.Fatalf("error message missing accepted model %q: %s", want, got)
}
Expand Down Expand Up @@ -171,7 +172,7 @@ func TestRunBenchmark_WhenSessionHasAPIUsage_ThenUsesTokenCountingAPIForNewConte
if countedText == "" {
t.Fatal("countTokensFn was not called")
}
wantCountModel := bm.TokenCountModelOpus48
wantCountModel := bm.TokenCountModelOpus55
if countModel != wantCountModel {
t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel)
}
Expand Down Expand Up @@ -328,7 +329,7 @@ func TestRunBenchmark_GivenSonnetModel_ThenUsesSonnetTokenCounterModel(t *testin
t.Fatalf("runBenchmark returned error: %v", err)
}

wantCountModel := bm.TokenCountModelSonnet
wantCountModel := bm.TokenCountModelSonnet55
if countModel != wantCountModel {
t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel)
}
Expand Down
12 changes: 7 additions & 5 deletions docs/benchmark.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ All other parameters are derived automatically from your real session data.
| `--days` | 30 | How far back to scan for sessions |
| `--min-kb` | 100 | Minimum JSONL file size in KB |
| `--n` | 10 | Max successful session results to report |
| `--model` | opus | Pricing and token-counting model: `opus`, `opus-4-6`, `opus-4-7`, `opus-4-8`, `opus-5-5`, `sonnet`, or `fable` (`fable-5-1`) |
| `--model` | opus | Pricing and token-counting model: `opus` (`opus-5-5`), `opus-4-6`, `opus-4-7`, `opus-4-8`, `sonnet` (`sonnet-5-5`), `sonnet-4-6`, or `fable` (`fable-5-1`) |

### Example output

Expand Down Expand Up @@ -162,17 +162,19 @@ the historical one-shot `NewCtx × CacheWrite` behavior.

`X` comes from transcript API usage and `C` comes from the Anthropic token
counting API. The `--model` flag controls both pricing and the tokenizer used by
the token counting API. `opus` is an alias for `opus-4-8`; explicit Opus 4.x
versions map to `claude-opus-4-6`, `claude-opus-4-7`, or `claude-opus-4-8`;
`opus-5-5` maps to `claude-opus-5-5`; `sonnet` maps to `claude-sonnet-4-6`;
the token counting API. `opus` and `opus-5-5` both map to `claude-opus-5-5`;
explicit Opus 4.x versions map to `claude-opus-4-6`, `claude-opus-4-7`, or
`claude-opus-4-8`. `sonnet` and `sonnet-5-5` both map to `claude-sonnet-5-5`
($2.00/M input, $0.20/M cache read, $2.50/M cache write, $10/M output);
`sonnet-4-6` maps to `claude-sonnet-4-6`.
`fable` and `fable-5-1` both map to `claude-fable-5-1`.
Opus 4.6, 4.7, and 4.8 use the same Opus pricing rates; Opus 5.5 has its own,
lower rates (see the pricing table above).
Fallback constants are used only for behavior that cannot be read directly from
transcript usage, such as sparse tool I/O data.

Cache read as a percentage of base input differs by model: Opus 4.x and Sonnet
use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers
(4.6 and 5.5) use 10%, Opus 5.5 uses 5%, and Claude Fable 5.1 uses 2.5%. Cost-savings numbers
are only directly comparable across rows using the same ratio.

### Simplifications
Expand Down
34 changes: 21 additions & 13 deletions internal/benchmark/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,13 @@ type Pricing struct {
BaseInput float64 // $/M tokens (uncached, after last breakpoint)
}

// PricingOpus is the shared Opus 4.x pricing (4.6, 4.7, 4.8).
var PricingOpus = Pricing{CachedRead: 0.50, CacheWrite: 6.25, BaseInput: 5.00}
var PricingSonnet = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00}
var PricingSonnet46 = Pricing{CachedRead: 0.30, CacheWrite: 3.75, BaseInput: 3.00}

// PricingSonnet55 is Claude Sonnet 5.5's pricing ($2 input / $10 output per
// MTok). Cache read stays at 10% of base input, like Sonnet 4.6.
var PricingSonnet55 = Pricing{CachedRead: 0.20, CacheWrite: 2.50, BaseInput: 2.00}

// PricingOpus55 is Claude Opus 5.5's pricing. Its cache read is 5% of base
// input rather than the 10% Opus 4.x/Sonnet use (source: Anthropic pricing
Expand All @@ -26,12 +31,13 @@ var PricingOpus55 = Pricing{CachedRead: 0.20, CacheWrite: 5.00, BaseInput: 4.00}
var PricingFable = Pricing{CachedRead: 0.25, CacheWrite: 12.50, BaseInput: 10.00}

const (
TokenCountModelOpus46 = "claude-opus-4-6"
TokenCountModelOpus47 = "claude-opus-4-7"
TokenCountModelOpus48 = "claude-opus-4-8"
TokenCountModelOpus55 = "claude-opus-5-5"
TokenCountModelSonnet = "claude-sonnet-4-6"
TokenCountModelFable = "claude-fable-5-1"
TokenCountModelOpus46 = "claude-opus-4-6"
TokenCountModelOpus47 = "claude-opus-4-7"
TokenCountModelOpus48 = "claude-opus-4-8"
TokenCountModelOpus55 = "claude-opus-5-5"
TokenCountModelSonnet46 = "claude-sonnet-4-6"
TokenCountModelSonnet55 = "claude-sonnet-5-5"
TokenCountModelFable = "claude-fable-5-1"
)

// ModelConfig bundles the pricing and token-counting model for a given model alias.
Expand All @@ -43,20 +49,22 @@ type ModelConfig struct {
// ResolveModel maps a user-facing model alias to its ModelConfig.
func ResolveModel(model string) (ModelConfig, error) {
switch model {
case "sonnet":
return ModelConfig{Pricing: PricingSonnet, TokenCountModel: TokenCountModelSonnet}, nil
case "opus", "opus-4-8":
case "sonnet", "sonnet-5-5":
return ModelConfig{Pricing: PricingSonnet55, TokenCountModel: TokenCountModelSonnet55}, nil
case "sonnet-4-6":
return ModelConfig{Pricing: PricingSonnet46, TokenCountModel: TokenCountModelSonnet46}, nil
case "opus", "opus-5-5":
return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil
case "opus-4-8":
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus48}, nil
case "opus-4-7":
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus47}, nil
case "opus-4-6":
return ModelConfig{Pricing: PricingOpus, TokenCountModel: TokenCountModelOpus46}, nil
case "opus-5-5":
return ModelConfig{Pricing: PricingOpus55, TokenCountModel: TokenCountModelOpus55}, nil
case "fable", "fable-5-1":
return ModelConfig{Pricing: PricingFable, TokenCountModel: TokenCountModelFable}, nil
default:
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, fable, or fable-5-1", model)
return ModelConfig{}, fmt.Errorf("unknown model %q: must be opus, opus-4-6, opus-4-7, opus-4-8, opus-5-5, sonnet, sonnet-4-6, sonnet-5-5, fable, or fable-5-1", model)
}
}

Expand Down
20 changes: 16 additions & 4 deletions internal/benchmark/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,26 @@ func Test_ResolveModel_GivenKnownAlias_ThenReturnsExpectedConfig(t *testing.T) {
{
name: "sonnet",
alias: "sonnet",
wantPricing: PricingSonnet,
wantTokenCount: TokenCountModelSonnet,
wantPricing: PricingSonnet55,
wantTokenCount: TokenCountModelSonnet55,
},
{
name: "sonnet-4-6",
alias: "sonnet-4-6",
wantPricing: PricingSonnet46,
wantTokenCount: TokenCountModelSonnet46,
},
{
name: "sonnet-5-5",
alias: "sonnet-5-5",
wantPricing: PricingSonnet55,
wantTokenCount: TokenCountModelSonnet55,
},
{
name: "opus",
alias: "opus",
wantPricing: PricingOpus,
wantTokenCount: TokenCountModelOpus48,
wantPricing: PricingOpus55,
wantTokenCount: TokenCountModelOpus55,
},
{
name: "opus-4-8",
Expand Down
2 changes: 1 addition & 1 deletion internal/tokens/tokens.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ const (
baseRetryDelay = 500 * time.Millisecond
)

const DefaultCountTokensModel = "claude-sonnet-4-6"
const DefaultCountTokensModel = "claude-sonnet-5-5"

// CountTokensAPI calls the Anthropic count_tokens endpoint.
// Resolves the API key from: env ANTHROPIC_API_KEY → config file path in
Expand Down
Loading