18109 llm usage statistics

STATISTIC.md

@@ -0,0 +1,253 @@
+statistics API response notes:
+- all statistics below use only data not older than the last 5 months
+- `costs.1month` and `costs.5month` remain cumulative spend windows
+- distributions are now time-series points, not histograms
+- each point is calculated from unique user spend aggregated inside a calendar period
+- calendar boundaries are used:
+  - day: from start to end of day
+  - week: from start to end of ISO-style week (Monday to Sunday)
+  - month: from start to end of month
+- returned windows:
+  - `daily*Distribution`: one point per day for the last month
+  - `weekly*Distribution`: one point per week for the last 5 months
+  - `monthly*Distribution`: one point per month for the last 5 months
+
+cost:
+{
+  "1month": float,
+  "5month": float
+}
+
+costPack:
+{
+  "codioProvided": cost,
+  "codioSpecial": cost
+}
+
+dailyDistributionDataPoint:
+{
+  "maxUserSpend": float,
+  "medianUserSpend": float,
+  "avgUserSpend": float,
+  "p95UserSpend": float,
+  "p99UserSpend": float,
+  "date": "YYYY-MM-DD"
+}
+
+periodDistributionDataPoint:
+{
+  "maxUserSpend": float,
+  "medianUserSpend": float,
+  "avgUserSpend": float,
+  "p95UserSpend": float,
+  "p99UserSpend": float,
+  "datePeriod": string
+}
+
+-----------------------
+
+all:
+{
+  "total": costPack,
+  "orgs": [
+    {
+      "id": string,
+      "costs": costPack
+    }
+  ]
+}
+
+---------------------
+
+topFiveItem:
+{
+  "userId": string,
+  "spendLastMonth": float
+}
+
+org:
+{
+  "id": string,
+  "costs": costPack,
+  "courses": [
+    {
+      "id": string,
+      "costs": costPack
+    }
+  ],
+  "dailySpecialDistribution": [dailyDistributionDataPoint],
+  "weeklySpecialDistribution": [periodDistributionDataPoint],
+  "monthlySpecialDistribution": [periodDistributionDataPoint],
+  "dailyCodioProvidedDistribution": [dailyDistributionDataPoint],
+  "weeklyCodioProvidedDistribution": [periodDistributionDataPoint],
+  "monthlyCodioProvidedDistribution": [periodDistributionDataPoint],
+  "topFive": [topFiveItem]
+}
+
+---------------------
+
+course:
+{
+  "id": string,
+  "costs": costPack,
+  "dailySpecialDistribution": [dailyDistributionDataPoint],
+  "weeklySpecialDistribution": [periodDistributionDataPoint],
+  "monthlySpecialDistribution": [periodDistributionDataPoint],
+  "dailyCodioProvidedDistribution": [dailyDistributionDataPoint],
+  "weeklyCodioProvidedDistribution": [periodDistributionDataPoint],
+  "monthlyCodioProvidedDistribution": [periodDistributionDataPoint],
+  "topFive": [topFiveItem]
+}
+
+-----------------------------
+
+How to interpret the returned statistics
+
+General principles:
+- each point is calculated from unique users active in that calendar period
+- for one period, each user contributes one aggregated spend value
+- metrics are then calculated across the set of user totals for that period
+- if a period has no activity, the API still returns a point with zero values so the frontend can render a continuous series
+
+What `costs` means:
+- `costs.codioProvided.1month` — cumulative spend for codio-provided traffic over the last 1 month
+- `costs.codioProvided.5month` — cumulative spend for codio-provided traffic over the last 5 months
+- `costs.codioSpecial.1month` — cumulative spend for codio-special traffic over the last 1 month
+- `costs.codioSpecial.5month` — cumulative spend for codio-special traffic over the last 5 months
+- this block is useful for total budget visibility, while the distribution arrays are useful for trend analysis
+
+What each distribution point means:
+- `maxUserSpend` — the highest per-user spend in that day/week/month
+- `medianUserSpend` — 50% of active users in that period spent at or below this value
+- `avgUserSpend` — arithmetic average per-user spend in that period
+- `p95UserSpend` — 95% of active users in that period spent at or below this value
+- `p99UserSpend` — 99% of active users in that period spent at or below this value
+- `date` is used for daily points
+- `datePeriod` is used for weekly/monthly points
+- `topFive` is a companion list for the current scope and contains the top 5 users by spend for the last month
+
+Date labels:
+- daily uses `YYYY-MM-DD`
+- weekly uses `YYYY-MM-DD/YYYY-MM-DD` where the label represents week start and week end
+- monthly uses `YYYY-MM`
+
+Examples:
+- `dailySpecialDistribution[i]` shows one calendar day from the last month
+- `weeklySpecialDistribution[i]` shows one calendar week from the last 5 months
+- `monthlyCodioProvidedDistribution[i]` shows one calendar month from the last 5 months
+- `topFive[i]` shows one of the five users with the highest spend for the last month in the current org or course scope
+
+How to read `topFive`
+
+- `topFive` is calculated for the last month
+- it is scoped to the current entity:
+  - for org statistics, top users inside that organization
+  - for course statistics, top users inside that course
+- it is not split by `codio-special` or `codio-provided`; it reflects total spend in scope for the last month
+- it is useful for quickly identifying the most expensive users for investigation, outreach, or manual policy review
+
+How to read the charts
+
+Daily chart:
+- use it to see short-term volatility and spikes
+- `maxUserSpend` highlights strongest single-user bursts in a day
+- `medianUserSpend` and `avgUserSpend` show whether broad usage is rising or only a few users are spiking
+
+Weekly chart:
+- use it to see medium-term usage stabilization
+- compare weekly `p95UserSpend` and `p99UserSpend` across weeks
+- this is useful for tuning weekly guardrails
+
+Monthly chart:
+- use it for budget policy and allowance planning
+- monthly changes are less noisy and better reflect stable behavior
+- a rising monthly `medianUserSpend` means the typical user is genuinely spending more
+
+How to use these metrics for limits
+
+Daily limits:
+- look at `daily...Distribution`
+- a high `p99UserSpend` with a low median usually means a few strong outliers
+- useful for anti-spike or abuse protection
+
+Weekly limits:
+- look at `weekly...Distribution`
+- stable weekly `p95UserSpend` can guide soft-limit candidates
+- rising weekly `maxUserSpend` may justify a hard cap or alerts
+
+Monthly limits:
+- look at `monthly...Distribution`
+- this is the best signal for recurring allowance defaults
+- if monthly `medianUserSpend` stays low but `p99UserSpend` rises, the tail is getting heavier without broad adoption
+
+Practical recommendation flow
+
+1. Start with monthly series:
+- review the trend of `medianUserSpend`, `p95UserSpend`, and `p99UserSpend`
+- use this to set or revise monthly allowances
+
+2. Check weekly series:
+- see whether usage changes smoothly week to week or has temporary bursts
+- if weekly p95 stays stable but max jumps, use alerts before stricter limits
+
+3. Check daily series:
+- use it for operational safety and anomaly detection
+- daily max and p99 are especially useful for identifying runaway prompts or abuse
+
+Suggested interpretation patterns
+
+Pattern A: Low median, high max, high p99 only on a few days
+- normal usage is cheap
+- spikes are rare and sharp
+- recommended action:
+  - keep daily hard caps
+  - avoid lowering broad monthly limits unnecessarily
+
+Pattern B: Median and avg both trend upward over weeks and months
+- usage is growing across the user base, not just in outliers
+- recommended action:
+  - revise default weekly/monthly budgets upward if product usage is healthy
+
+Pattern C: Monthly p95 rises while median stays flat
+- most users are stable, but heavy-user tail is getting more expensive
+- recommended action:
+  - keep default limits, but strengthen tail controls for power users
+
+Pattern D: Weekly and monthly max rise together
+- expensive usage is not only a single-day anomaly
+- recommended action:
+  - investigate long-running high-cost users and review policy settings
+
+Limitations of the current model
+
+- these arrays show metric trends over time, not spend histograms by bucket
+- percentiles can be noisy when the number of active users in a period is low
+- daily values are naturally more volatile than weekly/monthly values
+- recommendations should still be combined with product and business context
+
+Suggested UI usage
+
+For each org or course, show:
+- a `topFive` table or side panel with:
+  - `userId`
+  - `spendLastMonth`
+- separate charts for:
+- daily special
+- weekly special
+- monthly special
+- daily codio-provided
+- weekly codio-provided
+- monthly codio-provided
+
+For each chart, plot one or more lines:
+- `medianUserSpend`
+- `p95UserSpend`
+- `p99UserSpend`
+- optionally `maxUserSpend`
+
+Suggested helper text:
+- "Daily p99 shows near-worst-case per-user spend for a single day"
+- "Weekly median shows the typical user spend for a full calendar week"
+- "Monthly p95 is a strong candidate input for limit policy reviews"
+
+This makes the dashboard useful for monitoring trend shifts, tuning limits, and spotting anomalous growth.

cmd/bricksllm/main.go

@@ -297,6 +297,14 @@ func main() {
 		log.Sugar().Fatalf("error connecting to secondary keys redis storage: %v", err)
 	}
 
+	statisticsRedisCache := redis.NewClient(defaultRedisOption(cfg, 13))
+
+	ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if err = statisticsRedisCache.Ping(ctx).Err(); err != nil {
+		log.Sugar().Fatalf("error connecting to statistics redis storage: %v", err)
+	}
+
 	rateLimitCache := redisStorage.NewCache(rateLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	costLimitCache := redisStorage.NewCache(costLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	costStorage := redisStorage.NewStore(costRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
@@ -312,22 +320,23 @@ func main() {
 	keysCache := redisStorage.NewKeysCache(keysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	secondaryKeysCache := redisStorage.NewSecondaryKeysCache(secondaryKeysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 	requestsLimitStorage := redisStorage.NewStore(requestsLimitRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
+	statisticsCache := redisStorage.NewStatisticCache(statisticsRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout)
 
-	encryptor, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience)
+	encrypt, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience)
 	if cfg.EnableEncrytion && err != nil {
 		log.Sugar().Fatalf("error creating encryption client: %v", err)
 	}
 	v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage, requestsLimitStorage)
 
 	m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache, secondaryKeysCache, requestsLimitStorage)
-	krm := manager.NewReportingManager(costStorage, store, store, v)
-	psm := manager.NewProviderSettingsManager(store, psCache, encryptor)
+	krm := manager.NewReportingManager(costStorage, store, store, v, statisticsCache)
+	psm := manager.NewProviderSettingsManager(store, psCache, encrypt)
 	cpm := manager.NewCustomProvidersManager(store, cpMemStore)
 	rm := manager.NewRouteManager(store, store, rMemStore, psm)
 	pm := manager.NewPolicyManager(store, rMemStore)
 	um := manager.NewUserManager(store, store)
 
-	as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.AdminPass, cfg.XCodioSignSecret)
+	as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um)
 	if err != nil {
 		log.Sugar().Fatalf("error creating admin http server: %v", err)
 	}
@@ -358,7 +367,7 @@ func main() {
 
 	rec := recorder.NewRecorder(costStorage, userCostStorage, costLimitCache, userCostLimitCache, ce, store, requestsLimitStorage)
 	rlm := manager.NewRateLimitManager(rateLimitCache, userRateLimitCache)
-	a := auth.NewAuthenticator(psm, m, rm, store, encryptor)
+	a := auth.NewAuthenticator(psm, m, rm, store, encrypt)
 
 	c := cache.NewCache(apiCache)
 

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/fatih/color v1.18.0
 	github.com/gin-gonic/gin v1.11.0
+	github.com/google/tink/go v1.7.0
 	github.com/google/uuid v1.6.0
 	github.com/joho/godotenv v1.5.1
 	github.com/lib/pq v1.10.9

go.sum

@@ -115,6 +115,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
 github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
+github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w=
+github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=