Skip to content

refactor: make internal/models the single source of truth for models #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mapleeeeeeeeeee merged 3 commits into from
Sep 28, 2026
Merged

refactor: make internal/models the single source of truth for models #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7a3a50e
refactor: make internal/models the single source of truth for models
claude Sep 28, 2026
92647b8
refactor: drop Pricing alias and DefaultCountTokensModel indirection
claude Sep 28, 2026
5fa0613
test(models): drop redundant round-trip resolve check
claude Sep 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions cmd/cc-session/benchmark.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/analyzer"
bm "github.com/Mapleeeeeeeeeee/cc-session-reader/internal/benchmark"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/inject"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/models"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/parser"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/session"
)
Expand All @@ -30,7 +31,7 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S
days := fs.Int("days", 30, "how far back to scan")
minKB := fs.Int("min-kb", 100, "minimum JSONL file size in KB")
maxN := fs.Int("n", 10, "max sessions to include")
model := fs.String("model", "opus", "model: opus (opus-5-5), opus-4-6, opus-4-7, opus-4-8, sonnet (sonnet-5-5), sonnet-4-6, or fable (fable-5-1)")
model := fs.String("model", models.DefaultBenchmark.Alias(), "model: "+models.Usage())
overhead := fs.Int("overhead", 0, "session overhead tokens (system+tools+CLAUDE.md); measure with a 1-turn session")
isNoAPI := fs.Bool("no-api", false, "skip API calls; estimate filtered-text tokens with chars/2 (offline fallback)")
if err := fs.Parse(reorderArgs(args)); err != nil {
Expand All @@ -47,12 +48,12 @@ func runBenchmark(args []string, out io.Writer, errOut io.Writer, store parser.S
overheadToks = bm.DefaultOverhead
}

modelConfig, err := bm.ResolveModel(*model)
modelConfig, err := models.Resolve(*model)
if err != nil {
return err
}
p := modelConfig.Pricing
tokenCountModel := modelConfig.TokenCountModel
tokenCountModel := modelConfig.ID

all, _ := store.ListAllSessions()

Expand Down
49 changes: 3 additions & 46 deletions cmd/cc-session/benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/analyzer"
bm "github.com/Mapleeeeeeeeeee/cc-session-reader/internal/benchmark"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/inject"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/models"
"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/parser"
)

Expand Down Expand Up @@ -75,50 +76,6 @@ func TestPrintCompressionSection_GivenEvenCount_ThenPrintsAveragedMedian(t *test
}
}

func TestResolveBenchmarkModel_GivenAcceptedModelNames_ThenReturnsPricingAndTokenCountingModel(t *testing.T) {
tests := []struct {
name string
wantPricing bm.Pricing
wantTokenCountModel string
}{
{name: "opus", wantPricing: bm.PricingOpus55, wantTokenCountModel: bm.TokenCountModelOpus55},
{name: "opus-4-6", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus46},
{name: "opus-4-7", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus47},
{name: "opus-4-8", wantPricing: bm.PricingOpus, wantTokenCountModel: bm.TokenCountModelOpus48},
{name: "sonnet", wantPricing: bm.PricingSonnet55, wantTokenCountModel: bm.TokenCountModelSonnet55},
{name: "sonnet-4-6", wantPricing: bm.PricingSonnet46, wantTokenCountModel: bm.TokenCountModelSonnet46},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := bm.ResolveModel(tt.name)
if err != nil {
t.Fatalf("ResolveModel(%q) returned error: %v", tt.name, err)
}
if got.Pricing != tt.wantPricing {
t.Fatalf("pricing = %+v, want %+v", got.Pricing, tt.wantPricing)
}
if got.TokenCountModel != tt.wantTokenCountModel {
t.Fatalf("token count model = %q, want %q", got.TokenCountModel, tt.wantTokenCountModel)
}
})
}
}

func TestResolveBenchmarkModel_GivenUnknownModel_ThenReturnsAcceptedNames(t *testing.T) {
_, err := bm.ResolveModel("opus-4-5")
if err == nil {
t.Fatal("ResolveModel returned nil error for unknown model")
}

got := err.Error()
for _, want := range []string{"opus", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet", "sonnet-4-6"} {
if !strings.Contains(got, want) {
t.Fatalf("error message missing accepted model %q: %s", want, got)
}
}
}

func TestRunBenchmark_WhenSessionHasAPIUsage_ThenUsesTokenCountingAPIForNewContext(t *testing.T) {
root := t.TempDir()
sid := "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
Expand Down Expand Up @@ -172,7 +129,7 @@ func TestRunBenchmark_WhenSessionHasAPIUsage_ThenUsesTokenCountingAPIForNewConte
if countedText == "" {
t.Fatal("countTokensFn was not called")
}
wantCountModel := bm.TokenCountModelOpus55
wantCountModel := models.Opus55.ID
if countModel != wantCountModel {
t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel)
}
Expand Down Expand Up @@ -329,7 +286,7 @@ func TestRunBenchmark_GivenSonnetModel_ThenUsesSonnetTokenCounterModel(t *testin
t.Fatalf("runBenchmark returned error: %v", err)
}

wantCountModel := bm.TokenCountModelSonnet55
wantCountModel := models.Sonnet55.ID
if countModel != wantCountModel {
t.Fatalf("token counter model = %q, want %q", countModel, wantCountModel)
}
Expand Down
16 changes: 9 additions & 7 deletions internal/benchmark/cost.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package benchmark

import "github.com/Mapleeeeeeeeeee/cc-session-reader/internal/models"

// Cost model.
//
// Pricing: https://platform.claude.com/docs/en/build-with-claude/prompt-caching
Expand Down Expand Up @@ -95,7 +97,7 @@ func NewCostParams(r Result, overheadTokens int) CostParams {
//
// Prefix grows across turns by growth + (K-1)*toolIO per turn, because tool I/O
// from previous turns stays in the conversation history.
func CumulativeCostA(turns int, x int, sp CostParams, p Pricing) float64 {
func CumulativeCostA(turns int, x int, sp CostParams, p models.Pricing) float64 {
total := 0.0
extraCalls := extraCallsPerTurn(sp.K)
s := sp.ToolIOPerCall
Expand Down Expand Up @@ -125,7 +127,7 @@ func CumulativeCostA(turns int, x int, sp CostParams, p Pricing) float64 {
//
// Call 1: cache read (prefix from previous turn, or X when N=1) + cache write (R + P)
// Calls 2..K: cache read (growing prefix) + cache write (tool I/O)
func CumulativeCostAWarm(turns int, x int, sp CostParams, p Pricing) float64 {
func CumulativeCostAWarm(turns int, x int, sp CostParams, p models.Pricing) float64 {
total := 0.0
extraCalls := extraCallsPerTurn(sp.K)
s := sp.ToolIOPerCall
Expand All @@ -152,7 +154,7 @@ func CumulativeCostAWarm(turns int, x int, sp CostParams, p Pricing) float64 {
// Calls 2..K: cache read (growing) + cache write (tool I/O)
//
// Turn N (N>=2): same structure as A but with smaller base, cross-turn write = growth.
func CumulativeCostB(turns int, x int, filteredTokens int, sp CostParams, p Pricing) float64 {
func CumulativeCostB(turns int, x int, filteredTokens int, sp CostParams, p models.Pricing) float64 {
return CumulativeCostBWithInjectPages(turns, filteredTokens, 1, sp, p)
}

Expand All @@ -166,7 +168,7 @@ func CumulativeCostB(turns int, x int, filteredTokens int, sp CostParams, p Pric
// each page: read(overhead + previous page tokens) + write(page tokens)
//
// The active conversation after setup is still overhead + filteredTokens.
func CumulativeCostBWithInjectPages(turns int, filteredTokens int, injectPages int, sp CostParams, p Pricing) float64 {
func CumulativeCostBWithInjectPages(turns int, filteredTokens int, injectPages int, sp CostParams, p models.Pricing) float64 {
base := sp.Overhead + filteredTokens
total := newSessionSetupCost(filteredTokens, injectPages, sp, p)
extraCalls := extraCallsPerTurn(sp.K)
Expand All @@ -189,7 +191,7 @@ func CumulativeCostBWithInjectPages(turns int, filteredTokens int, injectPages i
return total
}

func newSessionSetupCost(filteredTokens int, injectPages int, sp CostParams, p Pricing) float64 {
func newSessionSetupCost(filteredTokens int, injectPages int, sp CostParams, p models.Pricing) float64 {
base := sp.Overhead + filteredTokens
if injectPages <= 1 {
return float64(base) * p.CacheWrite / 1e6
Expand All @@ -212,7 +214,7 @@ func extraCallsPerTurn(k float64) float64 {
return k - 1
}

func intraTurnToolCost(firstPrefix float64, extraCalls float64, toolIOPerCall int, p Pricing) float64 {
func intraTurnToolCost(firstPrefix float64, extraCalls float64, toolIOPerCall int, p models.Pricing) float64 {
if extraCalls <= 0 {
return 0
}
Expand All @@ -231,7 +233,7 @@ func intraTurnToolCost(firstPrefix float64, extraCalls float64, toolIOPerCall in
}

// ComputeCostMetrics populates the cost-related fields of r.
func ComputeCostMetrics(r *Result, overheadTokens int, p Pricing) {
func ComputeCostMetrics(r *Result, overheadTokens int, p models.Pricing) {
sp := NewCostParams(*r, overheadTokens)
r.BreakEven = -1
for n := 1; n <= 200; n++ {
Expand Down
49 changes: 27 additions & 22 deletions internal/benchmark/cost_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,15 @@ package benchmark
import (
"math"
"testing"

"github.com/Mapleeeeeeeeeee/cc-session-reader/internal/models"
)

const epsilon = 1e-9

// Expected values below assume the Opus 4.x rates ($5 / $6.25 / $0.50).
var testPricing = models.Opus48.Pricing

func approxEqual(got, want float64) bool {
return math.Abs(got-want) < epsilon
}
Expand Down Expand Up @@ -79,17 +84,17 @@ func Test_CumulativeCosts_GivenFractionalK_ThenUseFractionalCallsPerTurn(t *test
}{
{
name: "ColdExistingSession",
got: CumulativeCostA(1, 100000, testParamsK4478, PricingOpus),
got: CumulativeCostA(1, 100000, testParamsK4478, testPricing),
want: 0.9506535,
},
{
name: "WarmExistingSession",
got: CumulativeCostAWarm(1, 100000, testParamsK4478, PricingOpus),
got: CumulativeCostAWarm(1, 100000, testParamsK4478, testPricing),
want: 0.3916315,
},
{
name: "NewSession",
got: CumulativeCostB(1, 100000, 50000, testParamsK4478, PricingOpus),
got: CumulativeCostB(1, 100000, 50000, testParamsK4478, testPricing),
want: 0.9157635,
},
}
Expand Down Expand Up @@ -164,7 +169,7 @@ var costATests = []struct {
func Test_CumulativeCostA(t *testing.T) {
for _, tc := range costATests {
t.Run(tc.name, func(t *testing.T) {
got := CumulativeCostA(tc.turns, tc.x, tc.sp, PricingOpus)
got := CumulativeCostA(tc.turns, tc.x, tc.sp, testPricing)
if !approxEqual(got, tc.want) {
t.Errorf("CumulativeCostA(%d, %d) = %.10f, want %.10f", tc.turns, tc.x, got, tc.want)
}
Expand Down Expand Up @@ -225,7 +230,7 @@ var costAWarmTests = []struct {
func Test_CumulativeCostAWarm(t *testing.T) {
for _, tc := range costAWarmTests {
t.Run(tc.name, func(t *testing.T) {
got := CumulativeCostAWarm(tc.turns, tc.x, tc.sp, PricingOpus)
got := CumulativeCostAWarm(tc.turns, tc.x, tc.sp, testPricing)
if !approxEqual(got, tc.want) {
t.Errorf("CumulativeCostAWarm(%d, %d) = %.10f, want %.10f", tc.turns, tc.x, got, tc.want)
}
Expand Down Expand Up @@ -293,7 +298,7 @@ var costBTests = []struct {
func Test_CumulativeCostB(t *testing.T) {
for _, tc := range costBTests {
t.Run(tc.name, func(t *testing.T) {
got := CumulativeCostB(tc.turns, tc.x, tc.filteredTokens, tc.sp, PricingOpus)
got := CumulativeCostB(tc.turns, tc.x, tc.filteredTokens, tc.sp, testPricing)
if !approxEqual(got, tc.want) {
t.Errorf("CumulativeCostB(%d, %d, %d) = %.10f, want %.10f", tc.turns, tc.x, tc.filteredTokens, got, tc.want)
}
Expand All @@ -302,8 +307,8 @@ func Test_CumulativeCostB(t *testing.T) {
}

func Test_CumulativeCostBWithInjectPages_GivenMultiplePages_ThenSetupCostsMoreThanOneShot(t *testing.T) {
oneShot := CumulativeCostBWithInjectPages(0, 50000, 1, testParams, PricingOpus)
multiPage := CumulativeCostBWithInjectPages(0, 50000, 5, testParams, PricingOpus)
oneShot := CumulativeCostBWithInjectPages(0, 50000, 1, testParams, testPricing)
multiPage := CumulativeCostBWithInjectPages(0, 50000, 5, testParams, testPricing)

if !approxEqual(oneShot, 0.5625) {
t.Fatalf("one-page setup cost = %.10f, want historical one-shot cost 0.5625", oneShot)
Expand All @@ -319,8 +324,8 @@ func Test_CumulativeCostBWithInjectPages_GivenMultiplePages_ThenSetupCostsMoreTh
func Test_CumulativeCostAWarm_GivenAnyTurns_AlwaysCheaperThanColdCostA(t *testing.T) {
x := 100000
for turns := 1; turns <= 20; turns++ {
cold := CumulativeCostA(turns, x, testParams, PricingOpus)
warm := CumulativeCostAWarm(turns, x, testParams, PricingOpus)
cold := CumulativeCostA(turns, x, testParams, testPricing)
warm := CumulativeCostAWarm(turns, x, testParams, testPricing)
if warm >= cold {
t.Errorf("turns=%d: warm (%.10f) >= cold (%.10f); warm should always be cheaper", turns, warm, cold)
}
Expand All @@ -329,9 +334,9 @@ func Test_CumulativeCostAWarm_GivenAnyTurns_AlwaysCheaperThanColdCostA(t *testin

func Test_CumulativeCostA_GivenIncreasingTurns_ThenCostMonotonicallyIncreases(t *testing.T) {
x := 100000
prev := CumulativeCostA(1, x, testParams, PricingOpus)
prev := CumulativeCostA(1, x, testParams, testPricing)
for turns := 2; turns <= 20; turns++ {
curr := CumulativeCostA(turns, x, testParams, PricingOpus)
curr := CumulativeCostA(turns, x, testParams, testPricing)
if curr <= prev {
t.Errorf("turns=%d: cost (%.10f) not greater than turns=%d cost (%.10f)", turns, curr, turns-1, prev)
}
Expand All @@ -341,9 +346,9 @@ func Test_CumulativeCostA_GivenIncreasingTurns_ThenCostMonotonicallyIncreases(t

func Test_CumulativeCostAWarm_GivenIncreasingTurns_ThenCostMonotonicallyIncreases(t *testing.T) {
x := 100000
prev := CumulativeCostAWarm(1, x, testParams, PricingOpus)
prev := CumulativeCostAWarm(1, x, testParams, testPricing)
for turns := 2; turns <= 20; turns++ {
curr := CumulativeCostAWarm(turns, x, testParams, PricingOpus)
curr := CumulativeCostAWarm(turns, x, testParams, testPricing)
if curr <= prev {
t.Errorf("turns=%d: cost (%.10f) not greater than turns=%d cost (%.10f)", turns, curr, turns-1, prev)
}
Expand All @@ -356,8 +361,8 @@ func Test_CumulativeCostB_GivenSufficientTurns_ThenCheaperThanCostA(t *testing.T
filteredTokens := 50000 // base=90000 < x=100000 means compression happened
found := false
for turns := 1; turns <= 200; turns++ {
costA := CumulativeCostA(turns, x, testParams, PricingOpus)
costB := CumulativeCostB(turns, x, filteredTokens, testParams, PricingOpus)
costA := CumulativeCostA(turns, x, testParams, testPricing)
costB := CumulativeCostB(turns, x, filteredTokens, testParams, testPricing)
if costB < costA {
found = true
break
Expand All @@ -372,8 +377,8 @@ func Test_CumulativeCostB_GivenNoCompression_ThenNeverCheaperThanCostA(t *testin
x := 100000
filteredTokens := x // no compression: same as context
for turns := 1; turns <= 200; turns++ {
costA := CumulativeCostA(turns, x, testParams, PricingOpus)
costB := CumulativeCostB(turns, x, filteredTokens, testParams, PricingOpus)
costA := CumulativeCostA(turns, x, testParams, testPricing)
costB := CumulativeCostB(turns, x, filteredTokens, testParams, testPricing)
if costB < costA {
t.Errorf("turns=%d: CostB (%.10f) < CostA (%.10f) with no compression; setup cost should never be recovered", turns, costB, costA)
return
Expand Down Expand Up @@ -426,7 +431,7 @@ func Test_ComputeCostMetrics_GivenInjectPages_ThenColdAndWarmMetricsMatchFixture
InjectPages: tt.injectPages,
}

ComputeCostMetrics(&r, 40000, PricingOpus)
ComputeCostMetrics(&r, 40000, testPricing)

if r.BreakEven != tt.wantBreakEven {
t.Errorf("BreakEven = %d, want %d", r.BreakEven, tt.wantBreakEven)
Expand Down Expand Up @@ -459,7 +464,7 @@ func Test_ComputeCostMetrics_GivenCompression_ThenBreakEvenExists(t *testing.T)
AvgResponse: 2000,
Prompt: 10000,
}
ComputeCostMetrics(&r, 40000, PricingOpus)
ComputeCostMetrics(&r, 40000, testPricing)

if r.BreakEven <= 0 {
t.Errorf("BreakEven = %d, want > 0 when filteredTokens < contextTokens", r.BreakEven)
Expand All @@ -481,7 +486,7 @@ func Test_ComputeCostMetrics_GivenCompression_ThenWarmBreakEvenAtLeastBreakEven(
AvgResponse: 2000,
Prompt: 10000,
}
ComputeCostMetrics(&r, 40000, PricingOpus)
ComputeCostMetrics(&r, 40000, testPricing)

if r.WarmBreakEven != -1 && r.BreakEven != -1 && r.WarmBreakEven < r.BreakEven {
t.Errorf("WarmBreakEven (%d) < BreakEven (%d); warm cache is harder to beat, so WarmBreakEven must be >= BreakEven", r.WarmBreakEven, r.BreakEven)
Expand All @@ -497,7 +502,7 @@ func Test_ComputeCostMetrics_GivenNoCompression_ThenBreakEvenNeverReached(t *tes
AvgResponse: 2000,
Prompt: 10000,
}
ComputeCostMetrics(&r, 40000, PricingOpus)
ComputeCostMetrics(&r, 40000, testPricing)

if r.BreakEven != -1 {
t.Errorf("BreakEven = %d, want -1 when filteredTokens == contextTokens (no compression)", r.BreakEven)
Expand Down
Loading
Loading