From 4ec1c58e4b3f0adcd6931f829e333cb2812f8a1b Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 4 Sep 2026 13:34:31 +0100 Subject: [PATCH 01/14] wip --- internal/event/key_reporting.go | 57 ++++++++++++++++++++ internal/manager/reporting.go | 16 ++++++ internal/server/web/admin/admin.go | 3 ++ internal/server/web/admin/reporting.go | 72 ++++++++++++++++++++++++++ internal/storage/postgresql/event.go | 4 ++ 5 files changed, 152 insertions(+) diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index 6ebf3f4..1e1485d 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -1,5 +1,9 @@ package event +import ( + internalErrors "github.com/bricks-cloud/bricksllm/internal/errors" +) + type KeyDataPoint struct { KeyId string `json:"keyId"` CostInUsd float64 `json:"costInUsd"` @@ -76,3 +80,56 @@ type UsageData struct { type UsageReportingResponse struct { UsageData *UsageData `json:"usageData"` } + +type StatisticsRequest struct { + Level string `json:"level"` // all | org | course + Id *string `json:"id"` +} + +type StatisticLevel string + +var StisticLevels = struct { + Unknown StatisticLevel + All StatisticLevel + Org StatisticLevel + Course StatisticLevel + +}{ + Unknown: "unknown", + All: "all", + Org: "org", + Course: "course", +} + +func StatisticLevelFromStr(s string) StatisticLevel { + switch s { + case "all": + return StisticLevels.All + case "org": + return StisticLevels.Org + case "course": + return StisticLevels.Course + default: + return StisticLevels.Unknown + } +} + +func (r *StatisticsRequest) Validate() error { + if r.Level != "all" && r.Level != "org" && r.Level != "course" { + return internalErrors.NewValidationError("level must be one of 'all', 'org', or 'course'") + } + if (r.Level == "org" || r.Level == "course") && r.Id == nil { + return internalErrors.NewValidationError("id must be provided when level is 'org' or 'course'") + } + return nil +} + +func (r *StatisticsRequest) GetLevel() StatisticLevel { + return StatisticLevelFromStr(r.Level) +} + +type StatisticsResponse struct { + StatisticsData *StatisticsData `json:"statisticsData"` +} + +type StatisticsData struct {} diff --git a/internal/manager/reporting.go b/internal/manager/reporting.go index ffe2578..ecccd3f 100644 --- a/internal/manager/reporting.go +++ b/internal/manager/reporting.go @@ -33,6 +33,7 @@ type eventStorage interface { GetTopKeyRingDataPoints(start, end int64, tags []string, order string, limit, offset int, revoked *bool, topBy string) ([]*event.KeyRingDataPoint, error) GetUsageData(tags []string) (*event.UsageData, error) + GetStatisticsData(level event.StatisticLevel, id *string) (*event.StatisticsData, error) } type ReportingManager struct { @@ -187,6 +188,21 @@ func (rm *ReportingManager) GetUsageReporting(r *event.UsageReportingRequest) (* }, nil } +func (rm *ReportingManager) GetStatistic(r *event.StatisticsRequest) (*event.StatisticsResponse, error) { + if r == nil { + return nil, internal_errors.NewValidationError("statistics request cannot be nil") + } + // todo + statistics, err := rm.es.GetStatisticsData(r.GetLevel(), r.Id) + if err != nil { + return nil, err + } + // todo + return &event.StatisticsResponse{ + StatisticsData: statistics, + }, nil +} + func (rm *ReportingManager) GetCustomIds(keyId string) ([]string, error) { return rm.es.GetCustomIds(keyId) } diff --git a/internal/server/web/admin/admin.go b/internal/server/web/admin/admin.go index e83e846..048d273 100644 --- a/internal/server/web/admin/admin.go +++ b/internal/server/web/admin/admin.go @@ -43,6 +43,7 @@ type KeyReportingManager interface { GetTopKeyRingReporting(r *event.KeyRingReportingRequest) (*event.KeyRingReportingResponse, error) GetSpentKeyReporting(r *event.SpentKeyReportingRequest) (*event.SpentKeyReportingResponse, error) GetUsageReporting(r *event.UsageReportingRequest) (*event.UsageReportingResponse, error) + GetStatistic(r *event.StatisticsRequest) (*event.StatisticsResponse, error) GetKeyReporting(keyId string) (*key.KeyReporting, error) GetEvents(userId, customId string, keyIds []string, start int64, end int64) ([]*event.Event, error) @@ -103,6 +104,8 @@ func NewAdminServer(log *zap.Logger, mode string, m KeyManager, krm KeyReporting router.POST("/api/reporting/spent-keys", getGetSpentKeyMetricsHandler(krm, prod)) router.POST("/api/reporting/usage", getGetUsageMetricsHandler(krm, prod)) + router.POST("/api/reporting/statistic", getGetStatisticHandler(krm, prod)) + router.GET("/api/reporting/custom-ids", getGetCustomIdsHandler(krm, prod)) router.PUT("/api/provider-settings", getCreateProviderSettingHandler(psm, prod)) diff --git a/internal/server/web/admin/reporting.go b/internal/server/web/admin/reporting.go index fdbdaaf..1a07df7 100644 --- a/internal/server/web/admin/reporting.go +++ b/internal/server/web/admin/reporting.go @@ -550,3 +550,75 @@ func getGetUsageMetricsHandler(m KeyReportingManager, prod bool) gin.HandlerFunc c.JSON(http.StatusOK, reportingResponse) } } + +func getGetStatisticHandler(m KeyReportingManager, prod bool) gin.HandlerFunc { + return func(c *gin.Context) { + log := util.GetLogFromCtx(c) + telemetry.Incr("bricksllm.admin.get_get_statistic_handler.requests", nil, 1) + + start := time.Now() + defer func() { + dur := time.Since(start) + telemetry.Timing("bricksllm.admin.get_get_statistic_handler.latency", dur, nil, 1) + }() + + path := "/api/reporting/statistic" + + if c == nil || c.Request == nil { + c.JSON(http.StatusInternalServerError, &ErrorResponse{ + Type: "/errors/empty-context", + Title: "context is empty error", + Status: http.StatusInternalServerError, + Detail: "gin context is empty", + Instance: path, + }) + return + } + + data, err := io.ReadAll(c.Request.Body) + if err != nil { + logError(log, "error when reading usage reporting request body", prod, err) + c.JSON(http.StatusInternalServerError, &ErrorResponse{ + Type: "/errors/request-body-read", + Title: "request body reader error", + Status: http.StatusInternalServerError, + Detail: err.Error(), + Instance: path, + }) + return + } + + request := &event.StatisticsRequest{} + err = json.Unmarshal(data, request) + if err != nil { + logError(log, "error when unmarshalling statistics request body", prod, err) + c.JSON(http.StatusInternalServerError, &ErrorResponse{ + Type: "/errors/json-unmarshal", + Title: "json unmarshaller error", + Status: http.StatusInternalServerError, + Detail: err.Error(), + Instance: path, + }) + return + } + + statisticResponse, err := m.GetStatistic(request) + if err != nil { + telemetry.Incr("bricksllm.admin.get_get_statistic_handler.get_statistic", nil, 1) + + logError(log, "error when getting top key ring reporting", prod, err) + c.JSON(http.StatusInternalServerError, &ErrorResponse{ + Type: "/errors/event-reporting-manager", + Title: "usage reporting error", + Status: http.StatusInternalServerError, + Detail: err.Error(), + Instance: path, + }) + return + } + + telemetry.Incr("bricksllm.admin.get_get_statistic_handler.success", nil, 1) + + c.JSON(http.StatusOK, statisticResponse) + } +} diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index c3877e6..77e73d9 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -651,6 +651,10 @@ func (s *Store) GetUsageData(tags []string) (*event.UsageData, error) { return data, nil } +func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*event.StatisticsData, error) { + return &event.StatisticsData{}, nil +} + func (s *Store) GetAggregatedEventByDayDataPoints(start, end int64, keyIds []string) ([]*event.DataPointV2, error) { conditionBlock := fmt.Sprintf("WHERE time_stamp >= %d AND time_stamp < %d ", start, end) if len(keyIds) != 0 { From 02a138658f10805e32bc01eb32dda279f39787c9 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Sat, 5 Sep 2026 22:20:54 +0100 Subject: [PATCH 02/14] wip --- TODO.md | 115 ++++++++ internal/event/key_reporting.go | 53 +++- internal/manager/reporting.go | 12 +- internal/storage/postgresql/event.go | 387 ++++++++++++++++++++++++++- 4 files changed, 561 insertions(+), 6 deletions(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..bfa69ee --- /dev/null +++ b/TODO.md @@ -0,0 +1,115 @@ +Вот подробное проектирование и структура дашборда для отслеживания статистики использования API-ключей. Проект разбит на 3 уровня детализации (Дриллдаун) и включает логику работы, описание экранов и структуру графиков.🗺️ Общая архитектура и логика переходовДашборд построен по принципу Top-Down (от общего к частному). Переход между уровнями происходит по клику на элемент таблицы или графика:Уровень 1: Организации (Orgs) ➡️ Уровень 2: Курсы (Courses) ➡️ Уровень 3: Пользователи (Users) + Аналитика ИИ.🖥️ Уровень 1: Главный экран — Статистика по Организациям (Per Org Usage)Логика: Самый верхний уровень. Показывает общую нагрузку на систему и распределение токенов/запросов между клиентами (B2B-организациями).Компоненты экрана:Суммарные KPI (Сверху): Общее число запросов, потрачено токенов, количество активных организаций за выбранный период (день/неделя/месяц).График «Топ-5 Организаций» (Слева): Горизонтальный Bar Chart (столбчатая диаграмма), чтобы сразу видеть главных потребителей лимитов.Главная таблица организаций (Справа/Снизу):Колонки: ID/Название организации, Всего запросов, Потрачено токенов (Prompt/Completion), Количество активных курсов, Лимит (%).Действие: Клик на строку организации открывает Уровень 2 для этой конкретной организации.🖥️ Уровень 2: Экран организации — Статистика по Курсам (Per Course Usage)Логика: Контекст сужается до одной выбранной организации. Помогает понять, какие именно образовательные программы внутри компании тратят ресурсы API.Компоненты экрана:Хлебные крошки (Breadcrumbs): Организации > Название Org А (кнопка назад).График «Динамика по курсам»: Накапливаемая диаграмма (Stacked Area Chart) по дням. Показывает, как росло потребление, и какой курс доминировал в конкретный день.Таблица курсов:Колонки: Название курса, Программа/Категория, Количество студентов, Суммарные токены, Ошибки API (%).Действие: Клик на курс «AI Literacy» (или любой другой) переводит на Уровень 3.🖥️ Уровень 3: Экран курса — Статистика по Пользователям (AI Literacy Course)Логика: Микро-уровень. Здесь вычисляются метрики вовлеченности студентов и выявляются «аномальные» пользователи (кто тратит слишком много или слишком мало). Для курса AI Literacy включается расширенный блок AI-аналитики.📈 Статистический блок (Расчет логики):Max User Usage: Максимальный объем токенов, потраченный одним самым активным студентом. Помогает выявить фрод, баги в промптах студентов или «накрутку».Median (Медиана): Значение, разделяющее студентов ровно пополам. Показывает реальное «среднее» потребление типичного студента, защищенное от влияния экстремальных выбросов (в отличие от среднего арифметического).Avg (Среднее арифметическое): Общие токены курса / Количество студентов.📊 Визуализация: Распределение (Bell Curve / График плотности)Для курса AI Literacy идеальным решением является кривая нормального распределения (Bell Curve). Она наглядно показывает, как распределилась нагрузка:Левый хвост: Студенты, которые почти не отправляли запросы (отстающие/невовлеченные).Пик (Центр): Основная масса студентов, уложившаяся в медианные значения.Правый хвост: Студенты с аномально высоким потреблением (ближе к Max Usage).Компоненты экрана:Виджеты метрик: Три карточки сверху: [ Max: 125k tokens ] | [ Median: 42k tokens ] | [ Avg: 48k tokens ].График распределения: Bell Curve (как на схеме выше) для экспресс-оценки вовлеченности.Таблица пользователей:Колонки: ID/Имя пользователя, Роль (Студент/Преподаватель), Количество запросов, Всего токенов, Изменение активности (%).💡 Технические рекомендации по реализации UIФильтры: На каждом уровне должен быть глобальный фильтр по датам (календарь) и типу токенов (Prompt vs Completion / Сash-запросы).Пагинация и поиск: Таблицы пользователей и организаций должны поддерживать серверную пагинацию, так как студентов на курсе могут быть тысячи.Экспорт: Добавьте кнопку «Скачать CSV» на уровне пользователей для передачи данных в академический отдел или бухгалтерию.Если вы хотите детализировать дизайн, уточните:Какая база данных или система логирования используется для сбора статистики?Планируется ли внедрение лимитов (Quotas) на уровне пользователя, которые нужно визуализировать (например, progress bar вокруг Max Usage)? + + +{ + "course_id": "course_ai_lit_2026", + "course_name": "AI Literacy Basics", + "org_id": "org_skolkovo_01", + "currency": "USD", + "time_frame": { + "start": "2026-08-01T00:00:00Z", + "end": "2026-08-31T23:59:59Z" + }, + "financial_kpis": { + "total_course_spend": 2450.75, + "max_user_spend": 145.20, + "median_user_spend": 18.50, + "avg_user_spend": 24.02 + }, + "bell_curve_data": [ + { "cost_bucket_usd": "0-5", "user_count": 15 }, + { "cost_bucket_usd": "5-15", "user_count": 45 }, + { "cost_bucket_usd": "15-25", "user_count": 120 }, + { "cost_bucket_usd": "25-40", "user_count": 85 }, + { "cost_bucket_usd": "40-70", "user_count": 30 }, + { "cost_bucket_usd": "70-100", "user_count": 8 }, + { "cost_bucket_usd": "100+", "user_count": 2 } + ], + "users_table": [ + { + "user_id": "user_student_442", + "name": "Иван Иванов", + "total_requests": 340, + "total_spend": 18.45, + "budget_limit_usd": 50.00, + "limit_used_percentage": 36.9 + }, + { + "user_id": "user_student_999", + "name": "Алексей Петров (Аномалия)", + "total_requests": 4500, + "total_spend": 145.20, + "budget_limit_usd": 150.00, + "limit_used_percentage": 96.8 + } + ] +} + +----------------------------------- + +cost: +{ + "1month": float, + "5month": float, +} + +costPack: +{ + "codioProvided": cost, + "codioSpecial": cost, +} + +bellCurveData: [ + { "costBucketUsd": "0-10", "userCount": 15, serialNo: 1 }, + { "costBucketUsd": "10-20", "userCount": 85, serialNo: 2 }, + { "costBucketUsd": "20-30", "userCount": 30, serialNo: 3 }, + { "costBucketUsd": "30-50", "userCount": 8, serialNo: 4 }, + { "costBucketUsd": "50+", "userCount": 2, serialNo: 5 } + ] + +financialKpis: { + "maxUserSpend": float, + "medianUserSpend": float, + "avgUserSpend": float + }, + +----------------------- + +all: +{ + "total": costPack, + "orgs": [ + { + "id": string, + "costs": costPack, + }, + ] +} + +--------------------- + +org: +{ + "id": string, + "costs": costPack, + "courses": [ + { + "id": string, + "costs": costPack, + }, + ], + "bellCurveSpecial": bellCurveData, + "financialKpisSpecial": financialKpis +} + +--------------------- + +course: +{ + "id": string, + "costs": costPack, + "bellCurveCodioProvided": bellCurveData, + "financialKpisCodioProvided": financialKpis +} + +----------------------------- diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index 1e1485d..ad35274 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -132,4 +132,55 @@ type StatisticsResponse struct { StatisticsData *StatisticsData `json:"statisticsData"` } -type StatisticsData struct {} +type StatisticsData struct { + AllStatisticsData *AllStatisticsData `json:"allStatisticsData,omitempty"` + OrgStatisticsData *OrgStatisticsData `json:"orgStatisticsData,omitempty"` + CourseStatisticsData *CourseStatisticsData `json:"courseStatisticsData,omitempty"` +} + +type Cost struct { + OneMonth float64 `json:"1month"` + FiveMonth float64 `json:"5month"` +} + +type CostPack struct { + CodioProvided Cost `json:"codioProvided"` + CodioSpecial Cost `json:"codioSpecial"` +} + +type BellCurveDataPoint struct { + CostBucketUsd string `json:"costBucketUsd"` + UserCount int `json:"userCount"` + SerialNo int `json:"serialNo"` +} + +type FinancialKpis struct { + MaxUserSpend float64 `json:"maxUserSpend"` + MedianUserSpend float64 `json:"medianUserSpend"` + AvgUserSpend float64 `json:"avgUserSpend"` +} + +type AllStatisticsData struct { + Total CostPack `json:"total"` + Orgs []ShortOrgStatisticsData `json:"orgs"` +} + +type ShortOrgStatisticsData struct { + Id string `json:"id"` + Costs CostPack `json:"costs"` +} + +type OrgStatisticsData struct { + Id string `json:"id"` + Costs CostPack `json:"costs"` + Courses []CourseStatisticsData `json:"courses"` + BellCurveSpecial []BellCurveDataPoint `json:"bellCurveSpecial"` + FinancialKpisSpecial FinancialKpis `json:"financialKpisSpecial"` +} + +type CourseStatisticsData struct { + Id string `json:"id"` + Costs CostPack `json:"costs"` + BellCurveCodioProvided []BellCurveDataPoint `json:"bellCurveCodioProvided"` + FinancialKpisCodioProvided FinancialKpis `json:"financialKpisCodioProvided"` +} diff --git a/internal/manager/reporting.go b/internal/manager/reporting.go index ecccd3f..596c42a 100644 --- a/internal/manager/reporting.go +++ b/internal/manager/reporting.go @@ -192,14 +192,18 @@ func (rm *ReportingManager) GetStatistic(r *event.StatisticsRequest) (*event.Sta if r == nil { return nil, internal_errors.NewValidationError("statistics request cannot be nil") } - // todo - statistics, err := rm.es.GetStatisticsData(r.GetLevel(), r.Id) + + if err := r.Validate(); err != nil { + return nil, err + } + + statisticsData, err := rm.es.GetStatisticsData(r.GetLevel(), r.Id) if err != nil { return nil, err } - // todo + return &event.StatisticsResponse{ - StatisticsData: statistics, + StatisticsData: statisticsData, }, nil } diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 77e73d9..def75dc 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -6,7 +6,9 @@ import ( "encoding/json" "errors" "fmt" + "math" "slices" + "sort" "strings" "time" @@ -652,7 +654,390 @@ func (s *Store) GetUsageData(tags []string) (*event.UsageData, error) { } func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*event.StatisticsData, error) { - return &event.StatisticsData{}, nil + result := &event.StatisticsData{} + + switch level { + case event.StisticLevels.All: + total, err := s.getTotalCostPack(nil) + if err != nil { + return nil, err + } + + orgPacks, err := s.GetOrgsCostPack() + if err != nil { + return nil, err + } + + orgs := make([]event.ShortOrgStatisticsData, 0, len(orgPacks)) + for orgID, costs := range orgPacks { + orgs = append(orgs, event.ShortOrgStatisticsData{Id: orgID, Costs: costs}) + } + slices.SortFunc(orgs, func(a, b event.ShortOrgStatisticsData) int { + return strings.Compare(a.Id, b.Id) + }) + + result.AllStatisticsData = &event.AllStatisticsData{ + Total: *total, + Orgs: orgs, + } + case event.StisticLevels.Org: + if id == nil || len(*id) == 0 { + return nil, internal_errors.NewValidationError("id must be provided when level is 'org'") + } + + costs, err := s.getTotalCostPack([]string{orgTagPrefix + *id}) + if err != nil { + return nil, err + } + + coursePacks, err := s.GetCoursesCostPack(*id) + if err != nil { + return nil, err + } + + courses := make([]event.CourseStatisticsData, 0, len(coursePacks)) + for courseID, courseCosts := range coursePacks { + courses = append(courses, event.CourseStatisticsData{Id: courseID, Costs: courseCosts}) + } + slices.SortFunc(courses, func(a, b event.CourseStatisticsData) int { + return strings.Compare(a.Id, b.Id) + }) + + bellCurve, kpis, err := s.getUserSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag) + if err != nil { + return nil, err + } + + result.OrgStatisticsData = &event.OrgStatisticsData{ + Id: *id, + Costs: *costs, + Courses: courses, + BellCurveSpecial: bellCurve, + FinancialKpisSpecial: *kpis, + } + case event.StisticLevels.Course: + if id == nil || len(*id) == 0 { + return nil, internal_errors.NewValidationError("id must be provided when level is 'course'") + } + + costs, err := s.GetCourseCostPack(*id) + if err != nil { + return nil, err + } + + bellCurve, kpis, err := s.getUserSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag) + if err != nil { + return nil, err + } + + result.CourseStatisticsData = &event.CourseStatisticsData{ + Id: *id, + Costs: *costs, + BellCurveCodioProvided: bellCurve, + FinancialKpisCodioProvided: *kpis, + } + default: + return nil, internal_errors.NewValidationError("invalid level") + } + + return result, nil +} + +// { +// "total": costPack, +// "orgs": [ +// { +// "id": string, +// "costs": costPack, +// }, +// ] +// } +// +// val ORG_TAG_PREFIX = "org-tag-" +// val USER_TAG_PREFIX = "user-tag-" +// val COURSE_TAG_PREFIX = "course-tag-" +// +// val CODIO_SPECIAL_TAG = "codio-special" +// val CODIO_PROVIDED_TAG = "codio-provided" +// +// type Cost struct { +// OneMonth float64 `json:"1month"` +// FiveMonth float64 `json:"5month"` +// } + +// type CostPack struct { +// CodioProvided Cost `json:"codioProvided"` +// CodioSpecial Cost `json:"codioSpecial"` +// } +// +// SELECT +// substring(elem FROM 'org-tag-(.+)') AS user_id, +// count(*) AS total_records +// FROM +// your_table, +// unnest(your_array_column) AS elem +// WHERE +// elem LIKE 'org-tag--%' +// GROUP BY +// user_id; +// +// +// + +// "event_id" "created_at" "tags" "key_id" "cost_in_usd" "provider" "model" "status_code" "prompt_token_count" "completion_token_count" "latency_in_ms" "path" "method" "custom_id" "request" "response" "user_id" "action" "policy_id" "route_id" "correlation_id" "metadata" +// "0df9bc37-0e76-4192-ad43-d721acf6d638" 1785927977 "{org-tag-134db24b-62c3-44f5-b929-ec668f13d98b,user-tag-00112233-4455-6677-9cef-5b5dd134bfbf,course-tag-44e1ee81b625dbb97e4b5f2c57ab3183,codio-special}" "a950fce4-f91a-4195-9202-24480956d2f7" 0 "openai" "gpt-5.4-nano" 401 0 0 266 "/api/providers/openai/v1/responses" "POST" "{""input"": [{""role"": ""user"", ""content"": [{""text"": ""hello"", ""type"": ""input_text""}]}], ""model"": ""gpt-5.4-nano"", ""stream"": true}" "{}" "6a797d0f-5480-4d8c-bf7a-931f97e00666" "{}" +// + + +const ( + orgTagPrefix = "org-tag-" + userTagPrefix = "user-tag-" + courseTagPrefix = "course-tag-" + codioSpecialTag = "codio-special" + codioProvidedTag = "codio-provided" +) + +func (s *Store) getGroupedCostPack(groupBy string, filterTags []string) (map[string]event.CostPack, error) { + now := time.Now() + oneMonthAgo := now.Add(-30 * 24 * time.Hour).Unix() + fiveMonthsAgo := now.Add(-5 * 30 * 24 * time.Hour).Unix() + + args := []any{groupBy, oneMonthAgo, fiveMonthsAgo} + conditions := []string{"e.created_at >= $3"} + index := 4 + + for _, tag := range filterTags { + conditions = append(conditions, fmt.Sprintf("e.tags @> $%d", index)) + args = append(args, pq.Array([]string{tag})) + index++ + } + + query := fmt.Sprintf(` + SELECT + regexp_replace(tag, '^' || $1, '') AS grouped_id, + COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.created_at >= $2 AND e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_one_month, + COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_five_month, + COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.created_at >= $2 AND e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_one_month, + COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_five_month + FROM events e, + LATERAL unnest(e.tags) AS tag + WHERE tag LIKE $1 || '%%' + AND %s + GROUP BY grouped_id + `, codioProvidedTag, codioProvidedTag, codioSpecialTag, codioSpecialTag, strings.Join(conditions, " AND ")) + + ctx, cancel := context.WithTimeout(context.Background(), s.rt) + defer cancel() + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + result := make(map[string]event.CostPack) + for rows.Next() { + var id string + var pack event.CostPack + + if err := rows.Scan( + &id, + &pack.CodioProvided.OneMonth, + &pack.CodioProvided.FiveMonth, + &pack.CodioSpecial.OneMonth, + &pack.CodioSpecial.FiveMonth, + ); err != nil { + return nil, err + } + + result[id] = pack + } + + return result, nil +} + +func (s *Store) getTotalCostPack(filterTags []string) (*event.CostPack, error) { + now := time.Now() + oneMonthAgo := now.Add(-30 * 24 * time.Hour).Unix() + fiveMonthsAgo := now.Add(-5 * 30 * 24 * time.Hour).Unix() + + args := []any{oneMonthAgo, fiveMonthsAgo} + conditions := []string{"created_at >= $2"} + index := 3 + + for _, tag := range filterTags { + conditions = append(conditions, fmt.Sprintf("tags @> $%d", index)) + args = append(args, pq.Array([]string{tag})) + index++ + } + + query := fmt.Sprintf(` + SELECT + COALESCE(SUM(cost_in_usd) FILTER (WHERE created_at >= $1 AND tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_one_month, + COALESCE(SUM(cost_in_usd) FILTER (WHERE tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_five_month, + COALESCE(SUM(cost_in_usd) FILTER (WHERE created_at >= $1 AND tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_one_month, + COALESCE(SUM(cost_in_usd) FILTER (WHERE tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_five_month + FROM events + WHERE %s + `, codioProvidedTag, codioProvidedTag, codioSpecialTag, codioSpecialTag, strings.Join(conditions, " AND ")) + + ctx, cancel := context.WithTimeout(context.Background(), s.rt) + defer cancel() + + pack := &event.CostPack{} + if err := s.db.QueryRowContext(ctx, query, args...).Scan( + &pack.CodioProvided.OneMonth, + &pack.CodioProvided.FiveMonth, + &pack.CodioSpecial.OneMonth, + &pack.CodioSpecial.FiveMonth, + ); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return pack, nil + } + return nil, err + } + + return pack, nil +} + +func (s *Store) GetOrgsCostPack() (map[string]event.CostPack, error) { + return s.getGroupedCostPack(orgTagPrefix, nil) +} + +func (s *Store) GetCoursesCostPack(orgId string) (map[string]event.CostPack, error) { + return s.getGroupedCostPack(courseTagPrefix, []string{orgTagPrefix + orgId}) +} + +func (s *Store) GetCourseCostPack(courseId string) (*event.CostPack, error) { + return s.getTotalCostPack([]string{courseTagPrefix + courseId}) +} + +func (s *Store) getUserSpendStatistics(filterTags []string, costTypeTag string) ([]event.BellCurveDataPoint, *event.FinancialKpis, error) { + spendByUser, err := s.getUserSpendByTags(filterTags, costTypeTag) + if err != nil { + return nil, nil, err + } + + values := make([]float64, 0, len(spendByUser)) + for _, spend := range spendByUser { + values = append(values, spend) + } + + bellCurve := buildBellCurve(values) + kpis := buildFinancialKpis(values) + return bellCurve, kpis, nil +} + +func (s *Store) getUserSpendByTags(filterTags []string, costTypeTag string) (map[string]float64, error) { + args := []any{userTagPrefix, costTypeTag} + conditions := []string{"tag LIKE $1 || '%'", "e.tags @> $2"} + index := 3 + + for _, tag := range filterTags { + conditions = append(conditions, fmt.Sprintf("e.tags @> $%d", index)) + args = append(args, pq.Array([]string{tag})) + index++ + } + + query := fmt.Sprintf(` + SELECT + regexp_replace(tag, '^' || $1, '') AS user_id, + COALESCE(SUM(e.cost_in_usd), 0) AS total_cost + FROM events e, + LATERAL unnest(e.tags) AS tag + WHERE %s + GROUP BY user_id + `, strings.Join(conditions, " AND ")) + + ctx, cancel := context.WithTimeout(context.Background(), s.rt) + defer cancel() + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + result := make(map[string]float64) + for rows.Next() { + var userID string + var spend float64 + if err := rows.Scan(&userID, &spend); err != nil { + return nil, err + } + result[userID] = spend + } + + return result, nil +} + +func buildBellCurve(values []float64) []event.BellCurveDataPoint { + buckets := []struct { + label string + min float64 + max float64 + }{ + {label: "0-10", min: 0, max: 10}, + {label: "10-20", min: 10, max: 20}, + {label: "20-30", min: 20, max: 30}, + {label: "30-50", min: 30, max: 50}, + {label: "50+", min: 50, max: math.Inf(1)}, + } + + counts := make([]int, len(buckets)) + for _, v := range values { + switch { + case v < 10: + counts[0]++ + case v < 20: + counts[1]++ + case v < 30: + counts[2]++ + case v < 50: + counts[3]++ + default: + counts[4]++ + } + } + + result := make([]event.BellCurveDataPoint, 0, len(buckets)) + for i, bucket := range buckets { + result = append(result, event.BellCurveDataPoint{ + CostBucketUsd: bucket.label, + UserCount: counts[i], + SerialNo: i + 1, + }) + } + + return result +} + +func buildFinancialKpis(values []float64) *event.FinancialKpis { + if len(values) == 0 { + return &event.FinancialKpis{} + } + + sorted := append([]float64(nil), values...) + sort.Float64s(sorted) + + total := 0.0 + for _, v := range sorted { + total += v + } + + median := 0.0 + mid := len(sorted) / 2 + if len(sorted)%2 == 0 { + median = (sorted[mid-1] + sorted[mid]) / 2 + } else { + median = sorted[mid] + } + + return &event.FinancialKpis{ + MaxUserSpend: sorted[len(sorted)-1], + MedianUserSpend: median, + AvgUserSpend: total / float64(len(sorted)), + } } func (s *Store) GetAggregatedEventByDayDataPoints(start, end int64, keyIds []string) ([]*event.DataPointV2, error) { From 4f281da6293e7b55d159d71835ef716f110cfa58 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Sat, 5 Sep 2026 22:36:37 +0100 Subject: [PATCH 03/14] wip --- TODO.md | 230 +++++++++++++++++++++++--- internal/event/key_reporting.go | 69 +++++--- internal/storage/postgresql/event.go | 233 ++++++++++++++++----------- 3 files changed, 393 insertions(+), 139 deletions(-) diff --git a/TODO.md b/TODO.md index bfa69ee..60a0b0b 100644 --- a/TODO.md +++ b/TODO.md @@ -47,31 +47,48 @@ ----------------------------------- +statistics API response notes: +- all period-based distributions and KPIs below are calculated only from data not older than the last 5 months +- `costs.1month` and `costs.5month` remain cumulative spend windows +- period distributions are built from aggregated `user-day`, `user-week`, and `user-month` spend values + cost: { "1month": float, - "5month": float, + "5month": float } costPack: { "codioProvided": cost, - "codioSpecial": cost, + "codioSpecial": cost } -bellCurveData: [ - { "costBucketUsd": "0-10", "userCount": 15, serialNo: 1 }, - { "costBucketUsd": "10-20", "userCount": 85, serialNo: 2 }, - { "costBucketUsd": "20-30", "userCount": 30, serialNo: 3 }, - { "costBucketUsd": "30-50", "userCount": 8, serialNo: 4 }, - { "costBucketUsd": "50+", "userCount": 2, serialNo: 5 } - ] +bellCurveDataPoint: +{ + "costBucketUsd": string, + "sampleCount": int, + "serialNo": int +} -financialKpis: { - "maxUserSpend": float, - "medianUserSpend": float, - "avgUserSpend": float - }, +financialKpis: +{ + "maxUserSpend": float, + "medianUserSpend": float, + "avgUserSpend": float, + "p90UserSpend": float, + "p95UserSpend": float, + "p99UserSpend": float, + "sampleCount": int, + "recommendedSoftLimitUsd": float, + "recommendedHardLimitUsd": float +} + +spendPeriodStatistics: +{ + "bellCurve": [bellCurveDataPoint], + "financialKpis": financialKpis +} ----------------------- @@ -81,8 +98,8 @@ all: "orgs": [ { "id": string, - "costs": costPack, - }, + "costs": costPack + } ] } @@ -95,21 +112,190 @@ org: "courses": [ { "id": string, - "costs": costPack, - }, + "costs": costPack + } ], - "bellCurveSpecial": bellCurveData, - "financialKpisSpecial": financialKpis + "dailySpecial": spendPeriodStatistics, + "weeklySpecial": spendPeriodStatistics, + "monthlySpecial": spendPeriodStatistics } +Recommended buckets for org special: +- daily: 0-1, 1-3, 3-5, 5-10, 10-20, 20+ +- weekly: 0-5, 5-10, 10-20, 20-50, 50-100, 100+ +- monthly: 0-10, 10-25, 25-50, 50-100, 100-250, 250+ + --------------------- course: { "id": string, "costs": costPack, - "bellCurveCodioProvided": bellCurveData, - "financialKpisCodioProvided": financialKpis + "dailyCodioProvided": spendPeriodStatistics, + "weeklyCodioProvided": spendPeriodStatistics, + "monthlyCodioProvided": spendPeriodStatistics } +Recommended buckets for course codio-provided: +- daily: 0-1, 1-3, 3-5, 5-10, 10-20, 20+ +- weekly: 0-5, 5-10, 10-20, 20-50, 50-100, 100+ +- monthly: 0-10, 10-25, 25-50, 50-100, 100-250, 250+ + ----------------------------- + +How to interpret the returned statistics + +General principles: +- all `daily*`, `weekly*`, and `monthly*` blocks are calculated only from events from the last 5 months +- each graph is built from aggregated spend per user per period, not from lifetime spend +- the backend first computes: + - one value per `user-day` + - one value per `user-week` + - one value per `user-month` +- then it builds distributions and KPIs from those values + +What `costs` means: +- `costs.codioProvided.1month` — cumulative spend for codio-provided traffic over the last 1 month +- `costs.codioProvided.5month` — cumulative spend for codio-provided traffic over the last 5 months +- `costs.codioSpecial.1month` — cumulative spend for codio-special traffic over the last 1 month +- `costs.codioSpecial.5month` — cumulative spend for codio-special traffic over the last 5 months +- this block is useful for total budget visibility, but by itself is not enough for choosing period-based limits + +What `spendPeriodStatistics` means: +- `bellCurve` shows the distribution of spend samples for a specific period +- `financialKpis` shows summary statistics over the same sample set +- `sampleCount` is the number of aggregated samples used for that period + +Example: +- if `dailySpecial.financialKpis.sampleCount = 1200` +- this means the dataset contains 1200 `user-day` spend values +- not necessarily 1200 unique users +- one active user can contribute multiple daily samples on different days + +How to read the bell curve / histogram + +For org-level `dailySpecial`: +- every sample is one user's total `codio-special` spend within one calendar day +- bucket `0-1` means: number of user-days where spend was less than 1 USD +- bucket `1-3` means: number of user-days where spend was from 1 USD to under 3 USD +- bucket `20+` means: number of user-days where spend was 20 USD or more + +For course-level `weeklyCodioProvided`: +- every sample is one user's total `codio-provided` spend within one calendar week +- if the rightmost buckets are growing, users increasingly hit high weekly cost ranges + +What the graph tells you: +- left-heavy distribution usually means most user-periods are cheap +- a long right tail means there are rare but expensive spikes +- if the center of mass shifts right over time, existing limits may become too strict +- if a large share of samples already sits near your current limit, many normal users may get blocked + +What each KPI means + +For every period block (`daily*`, `weekly*`, `monthly*`): +- `maxUserSpend` — the largest spend observed in one user-period sample +- `medianUserSpend` — the middle spend value; 50% of samples are below it +- `avgUserSpend` — arithmetic average across all samples +- `p90UserSpend` — 90% of samples are at or below this value +- `p95UserSpend` — 95% of samples are at or below this value +- `p99UserSpend` — 99% of samples are at or below this value +- `recommendedSoftLimitUsd` — current recommended soft threshold, equal to `p95` +- `recommendedHardLimitUsd` — current recommended hard threshold, equal to `p99 * 1.2` + +How to use these metrics for limits + +Daily limit: +- use `daily*` blocks +- good starting point: + - soft limit = `p95` + - hard limit = `p99 * 1.2` +- meaning: + - soft limit should catch unusually expensive daily behavior while affecting only a small minority of user-days + - hard limit should catch only strong outliers or suspicious spikes + +Weekly limit: +- use `weekly*` blocks +- useful when daily usage is noisy, but you want to control total weekly burn +- if weekly p95 is stable but daily max is noisy, a weekly limit may be more product-friendly than a strict daily cap + +Monthly limit: +- use `monthly*` blocks +- useful for budget governance and subscription-style controls +- if monthly p95 is close to business budget expectations, it is a strong candidate for the default monthly allowance + +Practical recommendation flow + +1. Start with daily statistics: +- inspect `daily*` bell curve +- check where most samples are concentrated +- compare current or planned daily limit with `p95` and `p99` + +2. Check weekly statistics: +- confirm that users with many moderate daily sessions do not accumulate into unexpectedly high weekly totals +- if weekly tail is too wide, add a weekly limit even if daily looks acceptable + +3. Check monthly statistics: +- make sure the monthly allowance matches your budget model +- monthly limit should reflect normal heavy users, not only median users + +4. Review tails: +- if `maxUserSpend` is much larger than `p99UserSpend`, the system has extreme outliers +- such cases often justify a hard limit, anomaly alert, or additional investigation + +Suggested interpretation patterns + +Pattern A: Most samples in the first buckets, low p95, very high max +- normal usage is cheap +- there are rare spikes or outliers +- recommended action: + - keep soft limit near p95 + - keep hard limit significantly above soft limit + - investigate top offenders separately + +Pattern B: Distribution centered close to the upper buckets +- many users naturally consume a lot +- low limits will likely block legitimate traffic +- recommended action: + - increase default limits + - use weekly/monthly controls instead of aggressive daily limits + +Pattern C: Daily looks healthy, weekly/monthly tails are wide +- individual days are fine, but sustained usage is expensive +- recommended action: + - keep daily limit moderate + - add stronger weekly/monthly guardrails + +Pattern D: p95 and p99 are both moving upward over time +- usage pattern is structurally changing +- recommended action: + - review limits periodically + - consider time-series tracking for p95/p99 as the next dashboard enhancement + +Limitations of the current model + +- the current API returns aggregated distributions and KPIs, not raw per-user samples +- recommendations are heuristic, not policy decisions +- `recommendedSoftLimitUsd` and `recommendedHardLimitUsd` are good defaults, but should still be reviewed against business rules +- if sample count is low, percentiles can be noisy; in that case, org-wide defaults or manual review may be safer + +Suggested UI usage + +For each org or course, show three sections: +- Daily +- Weekly +- Monthly + +Inside each section show: +- histogram from `bellCurve` +- KPI cards: + - median + - p95 + - p99 + - max + - recommended soft limit + - recommended hard limit +- optional helper text: + - "95% of user-periods are below X USD" + - "Only 1% of user-periods exceed Y USD" + +This makes the dashboard directly actionable for tuning quotas and budget limits. diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index ad35274..8adce2c 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -82,23 +82,22 @@ type UsageReportingResponse struct { } type StatisticsRequest struct { - Level string `json:"level"` // all | org | course - Id *string `json:"id"` + Level string `json:"level"` // all | org | course + Id *string `json:"id"` } type StatisticLevel string var StisticLevels = struct { Unknown StatisticLevel - All StatisticLevel - Org StatisticLevel - Course StatisticLevel - + All StatisticLevel + Org StatisticLevel + Course StatisticLevel }{ Unknown: "unknown", - All: "all", - Org: "org", - Course: "course", + All: "all", + Org: "org", + Course: "course", } func StatisticLevelFromStr(s string) StatisticLevel { @@ -133,13 +132,13 @@ type StatisticsResponse struct { } type StatisticsData struct { - AllStatisticsData *AllStatisticsData `json:"allStatisticsData,omitempty"` - OrgStatisticsData *OrgStatisticsData `json:"orgStatisticsData,omitempty"` + AllStatisticsData *AllStatisticsData `json:"allStatisticsData,omitempty"` + OrgStatisticsData *OrgStatisticsData `json:"orgStatisticsData,omitempty"` CourseStatisticsData *CourseStatisticsData `json:"courseStatisticsData,omitempty"` } type Cost struct { - OneMonth float64 `json:"1month"` + OneMonth float64 `json:"1month"` FiveMonth float64 `json:"5month"` } @@ -150,18 +149,29 @@ type CostPack struct { type BellCurveDataPoint struct { CostBucketUsd string `json:"costBucketUsd"` - UserCount int `json:"userCount"` + SampleCount int `json:"sampleCount"` SerialNo int `json:"serialNo"` } type FinancialKpis struct { - MaxUserSpend float64 `json:"maxUserSpend"` - MedianUserSpend float64 `json:"medianUserSpend"` - AvgUserSpend float64 `json:"avgUserSpend"` + MaxUserSpend float64 `json:"maxUserSpend"` + MedianUserSpend float64 `json:"medianUserSpend"` + AvgUserSpend float64 `json:"avgUserSpend"` + P90UserSpend float64 `json:"p90UserSpend"` + P95UserSpend float64 `json:"p95UserSpend"` + P99UserSpend float64 `json:"p99UserSpend"` + SampleCount int `json:"sampleCount"` + RecommendedSoftLimitUsd float64 `json:"recommendedSoftLimitUsd"` + RecommendedHardLimitUsd float64 `json:"recommendedHardLimitUsd"` +} + +type SpendPeriodStatistics struct { + BellCurve []BellCurveDataPoint `json:"bellCurve"` + FinancialKpis FinancialKpis `json:"financialKpis"` } type AllStatisticsData struct { - Total CostPack `json:"total"` + Total CostPack `json:"total"` Orgs []ShortOrgStatisticsData `json:"orgs"` } @@ -170,17 +180,24 @@ type ShortOrgStatisticsData struct { Costs CostPack `json:"costs"` } +type ShortCourseStatisticsData struct { + Id string `json:"id"` + Costs CostPack `json:"costs"` +} + type OrgStatisticsData struct { - Id string `json:"id"` - Costs CostPack `json:"costs"` - Courses []CourseStatisticsData `json:"courses"` - BellCurveSpecial []BellCurveDataPoint `json:"bellCurveSpecial"` - FinancialKpisSpecial FinancialKpis `json:"financialKpisSpecial"` + Id string `json:"id"` + Costs CostPack `json:"costs"` + Courses []ShortCourseStatisticsData `json:"courses"` + DailySpecial SpendPeriodStatistics `json:"dailySpecial"` + WeeklySpecial SpendPeriodStatistics `json:"weeklySpecial"` + MonthlySpecial SpendPeriodStatistics `json:"monthlySpecial"` } type CourseStatisticsData struct { - Id string `json:"id"` - Costs CostPack `json:"costs"` - BellCurveCodioProvided []BellCurveDataPoint `json:"bellCurveCodioProvided"` - FinancialKpisCodioProvided FinancialKpis `json:"financialKpisCodioProvided"` + Id string `json:"id"` + Costs CostPack `json:"costs"` + DailyCodioProvided SpendPeriodStatistics `json:"dailyCodioProvided"` + WeeklyCodioProvided SpendPeriodStatistics `json:"weeklyCodioProvided"` + MonthlyCodioProvided SpendPeriodStatistics `json:"monthlyCodioProvided"` } diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index def75dc..46b2fe4 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -695,25 +695,34 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even return nil, err } - courses := make([]event.CourseStatisticsData, 0, len(coursePacks)) + courses := make([]event.ShortCourseStatisticsData, 0, len(coursePacks)) for courseID, courseCosts := range coursePacks { - courses = append(courses, event.CourseStatisticsData{Id: courseID, Costs: courseCosts}) + courses = append(courses, event.ShortCourseStatisticsData{Id: courseID, Costs: courseCosts}) } - slices.SortFunc(courses, func(a, b event.CourseStatisticsData) int { + slices.SortFunc(courses, func(a, b event.ShortCourseStatisticsData) int { return strings.Compare(a.Id, b.Id) }) - bellCurve, kpis, err := s.getUserSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag) + dailySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "day") + if err != nil { + return nil, err + } + weeklySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "week") + if err != nil { + return nil, err + } + monthlySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "month") if err != nil { return nil, err } result.OrgStatisticsData = &event.OrgStatisticsData{ - Id: *id, - Costs: *costs, - Courses: courses, - BellCurveSpecial: bellCurve, - FinancialKpisSpecial: *kpis, + Id: *id, + Costs: *costs, + Courses: courses, + DailySpecial: *dailySpecial, + WeeklySpecial: *weeklySpecial, + MonthlySpecial: *monthlySpecial, } case event.StisticLevels.Course: if id == nil || len(*id) == 0 { @@ -725,16 +734,25 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even return nil, err } - bellCurve, kpis, err := s.getUserSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag) + dailyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "day") + if err != nil { + return nil, err + } + weeklyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "week") + if err != nil { + return nil, err + } + monthlyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "month") if err != nil { return nil, err } result.CourseStatisticsData = &event.CourseStatisticsData{ - Id: *id, - Costs: *costs, - BellCurveCodioProvided: bellCurve, - FinancialKpisCodioProvided: *kpis, + Id: *id, + Costs: *costs, + DailyCodioProvided: *dailyProvided, + WeeklyCodioProvided: *weeklyProvided, + MonthlyCodioProvided: *monthlyProvided, } default: return nil, internal_errors.NewValidationError("invalid level") @@ -752,14 +770,14 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even // }, // ] // } -// +// // val ORG_TAG_PREFIX = "org-tag-" // val USER_TAG_PREFIX = "user-tag-" // val COURSE_TAG_PREFIX = "course-tag-" -// +// // val CODIO_SPECIAL_TAG = "codio-special" // val CODIO_PROVIDED_TAG = "codio-provided" -// +// // type Cost struct { // OneMonth float64 `json:"1month"` // FiveMonth float64 `json:"5month"` @@ -769,32 +787,32 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even // CodioProvided Cost `json:"codioProvided"` // CodioSpecial Cost `json:"codioSpecial"` // } -// -// SELECT +// +// SELECT // substring(elem FROM 'org-tag-(.+)') AS user_id, // count(*) AS total_records -// FROM +// FROM // your_table, // unnest(your_array_column) AS elem -// WHERE +// WHERE // elem LIKE 'org-tag--%' -// GROUP BY +// GROUP BY // user_id; -// -// -// +// +// +// // "event_id" "created_at" "tags" "key_id" "cost_in_usd" "provider" "model" "status_code" "prompt_token_count" "completion_token_count" "latency_in_ms" "path" "method" "custom_id" "request" "response" "user_id" "action" "policy_id" "route_id" "correlation_id" "metadata" // "0df9bc37-0e76-4192-ad43-d721acf6d638" 1785927977 "{org-tag-134db24b-62c3-44f5-b929-ec668f13d98b,user-tag-00112233-4455-6677-9cef-5b5dd134bfbf,course-tag-44e1ee81b625dbb97e4b5f2c57ab3183,codio-special}" "a950fce4-f91a-4195-9202-24480956d2f7" 0 "openai" "gpt-5.4-nano" 401 0 0 266 "/api/providers/openai/v1/responses" "POST" "{""input"": [{""role"": ""user"", ""content"": [{""text"": ""hello"", ""type"": ""input_text""}]}], ""model"": ""gpt-5.4-nano"", ""stream"": true}" "{}" "6a797d0f-5480-4d8c-bf7a-931f97e00666" "{}" -// - +// const ( - orgTagPrefix = "org-tag-" - userTagPrefix = "user-tag-" - courseTagPrefix = "course-tag-" - codioSpecialTag = "codio-special" - codioProvidedTag = "codio-provided" + orgTagPrefix = "org-tag-" + userTagPrefix = "user-tag-" + courseTagPrefix = "course-tag-" + codioSpecialTag = "codio-special" + codioProvidedTag = "codio-provided" + statisticsLookbackAge = -5 * 30 * 24 * time.Hour ) func (s *Store) getGroupedCostPack(groupBy string, filterTags []string) (map[string]event.CostPack, error) { @@ -912,26 +930,32 @@ func (s *Store) GetCourseCostPack(courseId string) (*event.CostPack, error) { return s.getTotalCostPack([]string{courseTagPrefix + courseId}) } -func (s *Store) getUserSpendStatistics(filterTags []string, costTypeTag string) ([]event.BellCurveDataPoint, *event.FinancialKpis, error) { - spendByUser, err := s.getUserSpendByTags(filterTags, costTypeTag) +func (s *Store) getPeriodSpendStatistics(filterTags []string, costTypeTag, period string) (*event.SpendPeriodStatistics, error) { + values, err := s.getUserPeriodSpendValues(filterTags, costTypeTag, period) if err != nil { - return nil, nil, err + return nil, err } - values := make([]float64, 0, len(spendByUser)) - for _, spend := range spendByUser { - values = append(values, spend) + buckets, err := getBucketsForPeriod(period) + if err != nil { + return nil, err } - bellCurve := buildBellCurve(values) - kpis := buildFinancialKpis(values) - return bellCurve, kpis, nil + return &event.SpendPeriodStatistics{ + BellCurve: buildBellCurve(values, buckets), + FinancialKpis: *buildFinancialKpis(values), + }, nil } -func (s *Store) getUserSpendByTags(filterTags []string, costTypeTag string) (map[string]float64, error) { - args := []any{userTagPrefix, costTypeTag} - conditions := []string{"tag LIKE $1 || '%'", "e.tags @> $2"} - index := 3 +func (s *Store) getUserPeriodSpendValues(filterTags []string, costTypeTag, period string) ([]float64, error) { + if period != "day" && period != "week" && period != "month" { + return nil, internal_errors.NewValidationError("unsupported period") + } + + fiveMonthsAgo := time.Now().Add(statisticsLookbackAge).Unix() + args := []any{userTagPrefix, pq.Array([]string{costTypeTag}), fiveMonthsAgo} + conditions := []string{"tag LIKE $1 || '%'", "e.tags @> $2", "e.created_at >= $3"} + index := 4 for _, tag := range filterTags { conditions = append(conditions, fmt.Sprintf("e.tags @> $%d", index)) @@ -940,14 +964,19 @@ func (s *Store) getUserSpendByTags(filterTags []string, costTypeTag string) (map } query := fmt.Sprintf(` - SELECT - regexp_replace(tag, '^' || $1, '') AS user_id, - COALESCE(SUM(e.cost_in_usd), 0) AS total_cost - FROM events e, - LATERAL unnest(e.tags) AS tag - WHERE %s - GROUP BY user_id - `, strings.Join(conditions, " AND ")) + SELECT user_period_cost + FROM ( + SELECT + regexp_replace(tag, '^' || $1, '') AS user_id, + date_trunc('%s', to_timestamp(e.created_at)) AS period_start, + COALESCE(SUM(e.cost_in_usd), 0) AS user_period_cost + FROM events e, + LATERAL unnest(e.tags) AS tag + WHERE %s + GROUP BY user_id, period_start + ) AS aggregated_user_spend + ORDER BY period_start, user_id + `, period, strings.Join(conditions, " AND ")) ctx, cancel := context.WithTimeout(context.Background(), s.rt) defer cancel() @@ -958,45 +987,47 @@ func (s *Store) getUserSpendByTags(filterTags []string, costTypeTag string) (map } defer rows.Close() - result := make(map[string]float64) + values := []float64{} for rows.Next() { - var userID string var spend float64 - if err := rows.Scan(&userID, &spend); err != nil { + if err := rows.Scan(&spend); err != nil { return nil, err } - result[userID] = spend + values = append(values, spend) + } + if err := rows.Err(); err != nil { + return nil, err } - return result, nil + return values, nil } -func buildBellCurve(values []float64) []event.BellCurveDataPoint { - buckets := []struct { - label string - min float64 - max float64 - }{ - {label: "0-10", min: 0, max: 10}, - {label: "10-20", min: 10, max: 20}, - {label: "20-30", min: 20, max: 30}, - {label: "30-50", min: 30, max: 50}, - {label: "50+", min: 50, max: math.Inf(1)}, +type costBucket struct { + label string + max float64 +} + +func getBucketsForPeriod(period string) ([]costBucket, error) { + switch period { + case "day": + return []costBucket{{label: "0-1", max: 1}, {label: "1-3", max: 3}, {label: "3-5", max: 5}, {label: "5-10", max: 10}, {label: "10-20", max: 20}, {label: "20+", max: math.Inf(1)}}, nil + case "week": + return []costBucket{{label: "0-5", max: 5}, {label: "5-10", max: 10}, {label: "10-20", max: 20}, {label: "20-50", max: 50}, {label: "50-100", max: 100}, {label: "100+", max: math.Inf(1)}}, nil + case "month": + return []costBucket{{label: "0-10", max: 10}, {label: "10-25", max: 25}, {label: "25-50", max: 50}, {label: "50-100", max: 100}, {label: "100-250", max: 250}, {label: "250+", max: math.Inf(1)}}, nil + default: + return nil, internal_errors.NewValidationError("unsupported period") } +} +func buildBellCurve(values []float64, buckets []costBucket) []event.BellCurveDataPoint { counts := make([]int, len(buckets)) for _, v := range values { - switch { - case v < 10: - counts[0]++ - case v < 20: - counts[1]++ - case v < 30: - counts[2]++ - case v < 50: - counts[3]++ - default: - counts[4]++ + for i, bucket := range buckets { + if v < bucket.max { + counts[i]++ + break + } } } @@ -1004,7 +1035,7 @@ func buildBellCurve(values []float64) []event.BellCurveDataPoint { for i, bucket := range buckets { result = append(result, event.BellCurveDataPoint{ CostBucketUsd: bucket.label, - UserCount: counts[i], + SampleCount: counts[i], SerialNo: i + 1, }) } @@ -1025,19 +1056,39 @@ func buildFinancialKpis(values []float64) *event.FinancialKpis { total += v } - median := 0.0 - mid := len(sorted) / 2 - if len(sorted)%2 == 0 { - median = (sorted[mid-1] + sorted[mid]) / 2 - } else { - median = sorted[mid] - } + p95 := percentile(sorted, 0.95) + p99 := percentile(sorted, 0.99) return &event.FinancialKpis{ - MaxUserSpend: sorted[len(sorted)-1], - MedianUserSpend: median, - AvgUserSpend: total / float64(len(sorted)), + MaxUserSpend: sorted[len(sorted)-1], + MedianUserSpend: percentile(sorted, 0.50), + AvgUserSpend: total / float64(len(sorted)), + P90UserSpend: percentile(sorted, 0.90), + P95UserSpend: p95, + P99UserSpend: p99, + SampleCount: len(sorted), + RecommendedSoftLimitUsd: p95, + RecommendedHardLimitUsd: p99 * 1.2, + } +} + +func percentile(sorted []float64, p float64) float64 { + if len(sorted) == 0 { + return 0 + } + if len(sorted) == 1 { + return sorted[0] } + + position := p * float64(len(sorted)-1) + lower := int(math.Floor(position)) + upper := int(math.Ceil(position)) + if lower == upper { + return sorted[lower] + } + + weight := position - float64(lower) + return sorted[lower] + (sorted[upper]-sorted[lower])*weight } func (s *Store) GetAggregatedEventByDayDataPoints(start, end int64, keyIds []string) ([]*event.DataPointV2, error) { From bd595e86b0db307e2dce71fc69650b45668c90da Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Sat, 5 Sep 2026 22:46:25 +0100 Subject: [PATCH 04/14] mock --- mock-data/README.md | 137 ++++++++++ mock-data/generate_statistics_mock.go | 346 ++++++++++++++++++++++++++ 2 files changed, 483 insertions(+) create mode 100644 mock-data/README.md create mode 100644 mock-data/generate_statistics_mock.go diff --git a/mock-data/README.md b/mock-data/README.md new file mode 100644 index 0000000..521bb12 --- /dev/null +++ b/mock-data/README.md @@ -0,0 +1,137 @@ +# Mock data for statistics testing + +This directory contains a Go script for generating synthetic `events` rows in PostgreSQL for testing statistics endpoints and limit-tuning dashboards. + +## Files + +- `generate_statistics_mock.go` — inserts mock rows into the `events` table + +## What the generator creates + +The script generates realistic event traffic across: +- organizations +- courses +- users +- `codio-provided` and `codio-special` usage types + +Each inserted event includes these tags: +- `org-tag-...` +- `course-tag-...` +- `user-tag-...` +- `codio-provided` or `codio-special` +- `mock-generated` + +The `mock-generated` tag is added specifically so the data can be removed manually after testing. + +## Default connection + +By default the script connects to: + +```text +postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432 +``` + +This matches the local PostgreSQL config from `docker-compose.yml`. + +## Run + +From the project root: + +```bash +go run ./mock-data/generate_statistics_mock.go +``` + +Example with explicit parameters: + +```bash +go run ./mock-data/generate_statistics_mock.go \ + -dsn "postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432" \ + -orgs 3 \ + -courses-per-org 4 \ + -users-per-course 80 \ + -days 150 \ + -seed 42 \ + -verbose +``` + +## Flags + +- `-dsn` — PostgreSQL DSN +- `-orgs` — number of organizations to generate +- `-courses-per-org` — number of courses per organization +- `-users-per-course` — number of users per course +- `-days` — how many past days of events to generate +- `-seed` — random seed for reproducible output +- `-verbose` — print periodic progress logs + +## Important note about 5-month statistics window + +Yes — you can intentionally generate data for a period **longer than 5 months** to verify that the statistics code correctly excludes old data. + +For example: + +```bash +go run ./mock-data/generate_statistics_mock.go -days 240 +``` + +This will insert mock events going back roughly 8 months. + +The current statistics implementation: +- uses only the **last 5 months** for: + - `daily*` + - `weekly*` + - `monthly*` + - bell-curve distributions + - percentile-based KPIs and limit recommendations +- also uses the same 5-month window for `costs.*.5month` +- uses the last 1 month for `costs.*.1month` + +So generating `-days` greater than `150` is a valid and useful test scenario. + +## Recommended validation scenarios + +### 1. Normal dashboard test +Generate exactly the recent window: + +```bash +go run ./mock-data/generate_statistics_mock.go -days 150 +``` + +Use this to inspect typical dashboard output. + +### 2. Old-data exclusion test +Generate a wider history: + +```bash +go run ./mock-data/generate_statistics_mock.go -days 240 +``` + +Use this to verify that: +- daily / weekly / monthly statistics do not drift because of older records +- `5month` cost values ignore the oldest generated rows +- recommendations are based only on recent behavior + +### 3. Higher-volume stress test + +```bash +go run ./mock-data/generate_statistics_mock.go -orgs 8 -courses-per-org 10 -users-per-course 200 -days 180 +``` + +Use this to test: +- query performance +- histogram stability +- UI rendering under larger sample counts + +## Cleanup + +To remove only rows generated by this script: + +```sql +DELETE FROM events WHERE tags @> ARRAY['mock-generated']::varchar[]; +``` + +## Notes + +- The generator creates outliers on purpose so that `p95`, `p99`, `maxUserSpend`, and recommended soft/hard limits are meaningful. +- The script is intended for local/dev environments. +- If the `events` table does not exist yet, the script creates it and applies the table alteration used by the application. diff --git a/mock-data/generate_statistics_mock.go b/mock-data/generate_statistics_mock.go new file mode 100644 index 0000000..65f8f2f --- /dev/null +++ b/mock-data/generate_statistics_mock.go @@ -0,0 +1,346 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/lib/pq" + + "github.com/bricks-cloud/bricksllm/internal/storage/postgresql" +) + +const ( + mockGeneratedTag = "mock-generated" + orgTagPrefix = "org-tag-" + courseTagPrefix = "course-tag-" + userTagPrefix = "user-tag-" + codioSpecialTag = "codio-special" + codioProvidedTag = "codio-provided" +) + +type config struct { + dsn string + orgCount int + coursesPerOrg int + usersPerCourse int + days int + seed int64 + verbose bool +} + +type courseDef struct { + orgID string + courseID string + users []string +} + +func main() { + cfg := parseFlags() + + store, err := postgresql.NewStore(cfg.dsn, 30*time.Second, 30*time.Second) + if err != nil { + panic(err) + } + + if err := store.CreateEventsTable(); err != nil { + panic(err) + } + if err := store.AlterEventsTable(); err != nil { + panic(err) + } + + db, err := sql.Open("postgres", cfg.dsn) + if err != nil { + panic(err) + } + defer db.Close() + + rng := rand.New(rand.NewSource(cfg.seed)) + courses := buildTopology(cfg) + + inserted, err := seedEvents(db, rng, cfg, courses) + if err != nil { + panic(err) + } + + fmt.Printf("Inserted %d mock events into events table.\n", inserted) + fmt.Printf("All generated rows include tag %q.\n", mockGeneratedTag) + fmt.Printf("Cleanup SQL: DELETE FROM events WHERE tags @> ARRAY['%s']::varchar[];\n", mockGeneratedTag) +} + +func parseFlags() config { + cfg := config{} + flag.StringVar(&cfg.dsn, "dsn", "postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432", "PostgreSQL DSN") + flag.IntVar(&cfg.orgCount, "orgs", 3, "Number of organizations") + flag.IntVar(&cfg.coursesPerOrg, "courses-per-org", 4, "Number of courses per organization") + flag.IntVar(&cfg.usersPerCourse, "users-per-course", 80, "Number of users per course") + flag.IntVar(&cfg.days, "days", 150, "How many past days of events to generate") + flag.Int64Var(&cfg.seed, "seed", 42, "Random seed for reproducible data") + flag.BoolVar(&cfg.verbose, "verbose", false, "Print progress details") + flag.Parse() + + if cfg.orgCount <= 0 || cfg.coursesPerOrg <= 0 || cfg.usersPerCourse <= 0 || cfg.days <= 0 { + panic("orgs, courses-per-org, users-per-course and days must be positive") + } + + return cfg +} + +func buildTopology(cfg config) []courseDef { + courses := make([]courseDef, 0, cfg.orgCount*cfg.coursesPerOrg) + for orgIdx := 1; orgIdx <= cfg.orgCount; orgIdx++ { + orgID := fmt.Sprintf("mock-org-%02d", orgIdx) + for courseIdx := 1; courseIdx <= cfg.coursesPerOrg; courseIdx++ { + courseID := fmt.Sprintf("mock-course-%02d-%02d", orgIdx, courseIdx) + users := make([]string, 0, cfg.usersPerCourse) + for userIdx := 1; userIdx <= cfg.usersPerCourse; userIdx++ { + users = append(users, fmt.Sprintf("mock-user-%02d-%02d-%03d", orgIdx, courseIdx, userIdx)) + } + courses = append(courses, courseDef{orgID: orgID, courseID: courseID, users: users}) + } + } + return courses +} + +func seedEvents(db *sql.DB, rng *rand.Rand, cfg config, courses []courseDef) (int, error) { + const insertQuery = ` + INSERT INTO events ( + event_id, + created_at, + tags, + key_id, + cost_in_usd, + provider, + model, + status_code, + prompt_token_count, + completion_token_count, + latency_in_ms, + path, + method, + custom_id, + request, + response, + user_id, + action, + policy_id, + route_id, + correlation_id, + metadata + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22 + ) + ` + + tx, err := db.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + + stmt, err := tx.Prepare(insertQuery) + if err != nil { + return 0, err + } + defer stmt.Close() + + now := time.Now().UTC() + inserted := 0 + + for dayOffset := 0; dayOffset < cfg.days; dayOffset++ { + day := now.AddDate(0, 0, -dayOffset) + for _, course := range courses { + activityMultiplier := courseActivityMultiplier(course) + for userIndex, userID := range course.users { + if !isUserActiveOnDay(rng, dayOffset, activityMultiplier, userIndex) { + continue + } + + eventCount := sampleEventCount(rng, dayOffset, activityMultiplier, userIndex) + for eventIdx := 0; eventIdx < eventCount; eventIdx++ { + createdAt := sampleTimestamp(rng, day) + costTypeTag, cost := sampleCostProfile(rng, dayOffset, activityMultiplier, userIndex) + promptTokens, completionTokens := sampleTokenCounts(rng, costTypeTag, activityMultiplier) + latency := 200 + rng.Intn(1800) + tags := []string{ + orgTagPrefix + course.orgID, + courseTagPrefix + course.courseID, + userTagPrefix + userID, + costTypeTag, + mockGeneratedTag, + } + + keyID := fmt.Sprintf("mock-key-%s", course.orgID) + customID := fmt.Sprintf("mock-run-%s", day.Format("20060102")) + action := "mock-generate-statistics" + policyID := fmt.Sprintf("mock-policy-%s", course.orgID) + routeID := fmt.Sprintf("mock-route-%s", course.courseID) + correlationID := uuid.NewString() + requestJSON := fmt.Sprintf(`{"mock":true,"tag":"%s","courseId":"%s","userId":"%s"}`, + mockGeneratedTag, + course.courseID, + userID, + ) + responseJSON := `{"ok":true}` + metadataJSON := fmt.Sprintf(`{"generator":"mock-data/generate_statistics_mock.go","dayOffset":%d}`, dayOffset) + + if _, err := stmt.Exec( + uuid.NewString(), + createdAt.Unix(), + pq.Array(tags), + keyID, + cost, + "openai", + sampleModel(costTypeTag), + 200, + promptTokens, + completionTokens, + latency, + "/api/providers/openai/v1/responses", + "POST", + customID, + requestJSON, + responseJSON, + userID, + action, + policyID, + routeID, + correlationID, + metadataJSON, + ); err != nil { + return inserted, err + } + inserted++ + } + } + } + + if cfg.verbose && dayOffset%7 == 0 { + fmt.Printf("generated through day offset %d (%s), inserted=%d\n", dayOffset, day.Format("2006-01-02"), inserted) + } + } + + if err := tx.Commit(); err != nil { + return inserted, err + } + + return inserted, nil +} + +func courseActivityMultiplier(course courseDef) float64 { + switch { + case strings.HasSuffix(course.courseID, "01"): + return 0.8 + case strings.HasSuffix(course.courseID, "02"): + return 1.0 + case strings.HasSuffix(course.courseID, "03"): + return 1.3 + default: + return 1.6 + } +} + +func isUserActiveOnDay(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) bool { + base := 0.25 + multiplier*0.18 + if userIndex < 5 { + base += 0.20 + } + if userIndex > 50 { + base -= 0.08 + } + if dayOffset%7 == 0 || dayOffset%7 == 6 { + base *= 0.75 + } + if dayOffset > 120 { + base *= 0.90 + } + if base > 0.95 { + base = 0.95 + } + if base < 0.05 { + base = 0.05 + } + return rng.Float64() < base +} + +func sampleEventCount(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) int { + count := 1 + if rng.Float64() < 0.45*multiplier { + count++ + } + if rng.Float64() < 0.15*multiplier { + count += 1 + rng.Intn(2) + } + if userIndex < 3 && rng.Float64() < 0.20 { + count += 2 + rng.Intn(4) + } + if dayOffset%30 == 0 && rng.Float64() < 0.30 { + count += 2 + } + if count > 10 { + count = 10 + } + return count +} + +func sampleTimestamp(rng *rand.Rand, day time.Time) time.Time { + base := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, time.UTC) + return base.Add(time.Duration(rng.Intn(24)) * time.Hour). + Add(time.Duration(rng.Intn(60)) * time.Minute). + Add(time.Duration(rng.Intn(60)) * time.Second) +} + +func sampleCostProfile(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) (string, float64) { + costTypeTag := codioProvidedTag + if rng.Float64() < 0.28 { + costTypeTag = codioSpecialTag + } + + base := 0.08 + rng.Float64()*0.90 + if costTypeTag == codioSpecialTag { + base *= 1.7 + } + base *= multiplier + + if userIndex < 3 { + base *= 2.5 + } + if userIndex >= 3 && userIndex < 10 { + base *= 1.4 + } + + if dayOffset%14 == 0 && rng.Float64() < 0.18 { + base *= 4.0 + rng.Float64()*5.0 + } + if dayOffset%45 == 0 && rng.Float64() < 0.10 { + base *= 8.0 + rng.Float64()*8.0 + } + + return costTypeTag, round2(base) +} + +func sampleTokenCounts(rng *rand.Rand, costTypeTag string, multiplier float64) (int, int) { + prompt := 300 + rng.Intn(2200) + completion := 150 + rng.Intn(1800) + if costTypeTag == codioSpecialTag { + prompt = int(float64(prompt) * (1.2 + multiplier*0.2)) + completion = int(float64(completion) * (1.2 + multiplier*0.2)) + } + return prompt, completion +} + +func sampleModel(costTypeTag string) string { + if costTypeTag == codioSpecialTag { + return "gpt-5.4-mini" + } + return "gpt-5.4-nano" +} + +func round2(v float64) float64 { + return float64(int(v*100+0.5)) / 100 +} From 4eda8f203176af25dd4eca4f18e9e700a0ac6042 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Mon, 7 Sep 2026 14:57:12 +0100 Subject: [PATCH 05/14] fixes --- TODO.md | 3 --- internal/storage/postgresql/event.go | 18 +++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/TODO.md b/TODO.md index 60a0b0b..7212160 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,3 @@ -Вот подробное проектирование и структура дашборда для отслеживания статистики использования API-ключей. Проект разбит на 3 уровня детализации (Дриллдаун) и включает логику работы, описание экранов и структуру графиков.🗺️ Общая архитектура и логика переходовДашборд построен по принципу Top-Down (от общего к частному). Переход между уровнями происходит по клику на элемент таблицы или графика:Уровень 1: Организации (Orgs) ➡️ Уровень 2: Курсы (Courses) ➡️ Уровень 3: Пользователи (Users) + Аналитика ИИ.🖥️ Уровень 1: Главный экран — Статистика по Организациям (Per Org Usage)Логика: Самый верхний уровень. Показывает общую нагрузку на систему и распределение токенов/запросов между клиентами (B2B-организациями).Компоненты экрана:Суммарные KPI (Сверху): Общее число запросов, потрачено токенов, количество активных организаций за выбранный период (день/неделя/месяц).График «Топ-5 Организаций» (Слева): Горизонтальный Bar Chart (столбчатая диаграмма), чтобы сразу видеть главных потребителей лимитов.Главная таблица организаций (Справа/Снизу):Колонки: ID/Название организации, Всего запросов, Потрачено токенов (Prompt/Completion), Количество активных курсов, Лимит (%).Действие: Клик на строку организации открывает Уровень 2 для этой конкретной организации.🖥️ Уровень 2: Экран организации — Статистика по Курсам (Per Course Usage)Логика: Контекст сужается до одной выбранной организации. Помогает понять, какие именно образовательные программы внутри компании тратят ресурсы API.Компоненты экрана:Хлебные крошки (Breadcrumbs): Организации > Название Org А (кнопка назад).График «Динамика по курсам»: Накапливаемая диаграмма (Stacked Area Chart) по дням. Показывает, как росло потребление, и какой курс доминировал в конкретный день.Таблица курсов:Колонки: Название курса, Программа/Категория, Количество студентов, Суммарные токены, Ошибки API (%).Действие: Клик на курс «AI Literacy» (или любой другой) переводит на Уровень 3.🖥️ Уровень 3: Экран курса — Статистика по Пользователям (AI Literacy Course)Логика: Микро-уровень. Здесь вычисляются метрики вовлеченности студентов и выявляются «аномальные» пользователи (кто тратит слишком много или слишком мало). Для курса AI Literacy включается расширенный блок AI-аналитики.📈 Статистический блок (Расчет логики):Max User Usage: Максимальный объем токенов, потраченный одним самым активным студентом. Помогает выявить фрод, баги в промптах студентов или «накрутку».Median (Медиана): Значение, разделяющее студентов ровно пополам. Показывает реальное «среднее» потребление типичного студента, защищенное от влияния экстремальных выбросов (в отличие от среднего арифметического).Avg (Среднее арифметическое): Общие токены курса / Количество студентов.📊 Визуализация: Распределение (Bell Curve / График плотности)Для курса AI Literacy идеальным решением является кривая нормального распределения (Bell Curve). Она наглядно показывает, как распределилась нагрузка:Левый хвост: Студенты, которые почти не отправляли запросы (отстающие/невовлеченные).Пик (Центр): Основная масса студентов, уложившаяся в медианные значения.Правый хвост: Студенты с аномально высоким потреблением (ближе к Max Usage).Компоненты экрана:Виджеты метрик: Три карточки сверху: [ Max: 125k tokens ] | [ Median: 42k tokens ] | [ Avg: 48k tokens ].График распределения: Bell Curve (как на схеме выше) для экспресс-оценки вовлеченности.Таблица пользователей:Колонки: ID/Имя пользователя, Роль (Студент/Преподаватель), Количество запросов, Всего токенов, Изменение активности (%).💡 Технические рекомендации по реализации UIФильтры: На каждом уровне должен быть глобальный фильтр по датам (календарь) и типу токенов (Prompt vs Completion / Сash-запросы).Пагинация и поиск: Таблицы пользователей и организаций должны поддерживать серверную пагинацию, так как студентов на курсе могут быть тысячи.Экспорт: Добавьте кнопку «Скачать CSV» на уровне пользователей для передачи данных в академический отдел или бухгалтерию.Если вы хотите детализировать дизайн, уточните:Какая база данных или система логирования используется для сбора статистики?Планируется ли внедрение лимитов (Quotas) на уровне пользователя, которые нужно визуализировать (например, progress bar вокруг Max Usage)? - - { "course_id": "course_ai_lit_2026", "course_name": "AI Literacy Basics", diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 46b2fe4..78677fe 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -832,16 +832,16 @@ func (s *Store) getGroupedCostPack(groupBy string, filterTags []string) (map[str query := fmt.Sprintf(` SELECT - regexp_replace(tag, '^' || $1, '') AS grouped_id, + regexp_replace(tag.tag, '^' || $1, '') AS grouped_id, COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.created_at >= $2 AND e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_one_month, COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_provided_five_month, COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.created_at >= $2 AND e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_one_month, COALESCE(SUM(e.cost_in_usd) FILTER (WHERE e.tags @> ARRAY['%s']::varchar[]), 0) AS codio_special_five_month FROM events e, - LATERAL unnest(e.tags) AS tag - WHERE tag LIKE $1 || '%%' + LATERAL unnest(e.tags) AS tag(tag) + WHERE tag.tag LIKE $1 || '%%' AND %s - GROUP BY grouped_id + GROUP BY regexp_replace(tag.tag, '^' || $1, '') `, codioProvidedTag, codioProvidedTag, codioSpecialTag, codioSpecialTag, strings.Join(conditions, " AND ")) ctx, cancel := context.WithTimeout(context.Background(), s.rt) @@ -954,7 +954,7 @@ func (s *Store) getUserPeriodSpendValues(filterTags []string, costTypeTag, perio fiveMonthsAgo := time.Now().Add(statisticsLookbackAge).Unix() args := []any{userTagPrefix, pq.Array([]string{costTypeTag}), fiveMonthsAgo} - conditions := []string{"tag LIKE $1 || '%'", "e.tags @> $2", "e.created_at >= $3"} + conditions := []string{"tag.tag LIKE $1 || '%'", "e.tags @> $2", "e.created_at >= $3"} index := 4 for _, tag := range filterTags { @@ -967,16 +967,16 @@ func (s *Store) getUserPeriodSpendValues(filterTags []string, costTypeTag, perio SELECT user_period_cost FROM ( SELECT - regexp_replace(tag, '^' || $1, '') AS user_id, + regexp_replace(tag.tag, '^' || $1, '') AS user_id, date_trunc('%s', to_timestamp(e.created_at)) AS period_start, COALESCE(SUM(e.cost_in_usd), 0) AS user_period_cost FROM events e, - LATERAL unnest(e.tags) AS tag + LATERAL unnest(e.tags) AS tag(tag) WHERE %s - GROUP BY user_id, period_start + GROUP BY regexp_replace(tag.tag, '^' || $1, ''), date_trunc('%s', to_timestamp(e.created_at)) ) AS aggregated_user_spend ORDER BY period_start, user_id - `, period, strings.Join(conditions, " AND ")) + `, period, strings.Join(conditions, " AND "), period) ctx, cancel := context.WithTimeout(context.Background(), s.rt) defer cancel() From 360e130a6371c4fe81cc3d1d9d661177184ce35b Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 9 Sep 2026 12:38:03 +0100 Subject: [PATCH 06/14] stats --- STATISTIC.md | 253 ++++++++++++++++++ TODO.md | 298 --------------------- internal/event/key_reporting.go | 60 +++-- internal/storage/postgresql/event.go | 371 ++++++++++++++++++--------- 4 files changed, 546 insertions(+), 436 deletions(-) create mode 100644 STATISTIC.md delete mode 100644 TODO.md diff --git a/STATISTIC.md b/STATISTIC.md new file mode 100644 index 0000000..435f11f --- /dev/null +++ b/STATISTIC.md @@ -0,0 +1,253 @@ +statistics API response notes: +- all statistics below use only data not older than the last 5 months +- `costs.1month` and `costs.5month` remain cumulative spend windows +- distributions are now time-series points, not histograms +- each point is calculated from unique user spend aggregated inside a calendar period +- calendar boundaries are used: + - day: from start to end of day + - week: from start to end of ISO-style week (Monday to Sunday) + - month: from start to end of month +- returned windows: + - `daily*Distribution`: one point per day for the last month + - `weekly*Distribution`: one point per week for the last 5 months + - `monthly*Distribution`: one point per month for the last 5 months + +cost: +{ + "1month": float, + "5month": float +} + +costPack: +{ + "codioProvided": cost, + "codioSpecial": cost +} + +dailyDistributionDataPoint: +{ + "maxUserSpend": float, + "medianUserSpend": float, + "avgUserSpend": float, + "p95UserSpend": float, + "p99UserSpend": float, + "date": "YYYY-MM-DD" +} + +periodDistributionDataPoint: +{ + "maxUserSpend": float, + "medianUserSpend": float, + "avgUserSpend": float, + "p95UserSpend": float, + "p99UserSpend": float, + "datePeriod": string +} + +----------------------- + +all: +{ + "total": costPack, + "orgs": [ + { + "id": string, + "costs": costPack + } + ] +} + +--------------------- + +topFiveItem: +{ + "userId": string, + "spendLastMonth": float +} + +org: +{ + "id": string, + "costs": costPack, + "courses": [ + { + "id": string, + "costs": costPack + } + ], + "dailySpecialDistribution": [dailyDistributionDataPoint], + "weeklySpecialDistribution": [periodDistributionDataPoint], + "monthlySpecialDistribution": [periodDistributionDataPoint], + "dailyCodioProvidedDistribution": [dailyDistributionDataPoint], + "weeklyCodioProvidedDistribution": [periodDistributionDataPoint], + "monthlyCodioProvidedDistribution": [periodDistributionDataPoint], + "topFive": [topFiveItem] +} + +--------------------- + +course: +{ + "id": string, + "costs": costPack, + "dailySpecialDistribution": [dailyDistributionDataPoint], + "weeklySpecialDistribution": [periodDistributionDataPoint], + "monthlySpecialDistribution": [periodDistributionDataPoint], + "dailyCodioProvidedDistribution": [dailyDistributionDataPoint], + "weeklyCodioProvidedDistribution": [periodDistributionDataPoint], + "monthlyCodioProvidedDistribution": [periodDistributionDataPoint], + "topFive": [topFiveItem] +} + +----------------------------- + +How to interpret the returned statistics + +General principles: +- each point is calculated from unique users active in that calendar period +- for one period, each user contributes one aggregated spend value +- metrics are then calculated across the set of user totals for that period +- if a period has no activity, the API still returns a point with zero values so the frontend can render a continuous series + +What `costs` means: +- `costs.codioProvided.1month` — cumulative spend for codio-provided traffic over the last 1 month +- `costs.codioProvided.5month` — cumulative spend for codio-provided traffic over the last 5 months +- `costs.codioSpecial.1month` — cumulative spend for codio-special traffic over the last 1 month +- `costs.codioSpecial.5month` — cumulative spend for codio-special traffic over the last 5 months +- this block is useful for total budget visibility, while the distribution arrays are useful for trend analysis + +What each distribution point means: +- `maxUserSpend` — the highest per-user spend in that day/week/month +- `medianUserSpend` — 50% of active users in that period spent at or below this value +- `avgUserSpend` — arithmetic average per-user spend in that period +- `p95UserSpend` — 95% of active users in that period spent at or below this value +- `p99UserSpend` — 99% of active users in that period spent at or below this value +- `date` is used for daily points +- `datePeriod` is used for weekly/monthly points +- `topFive` is a companion list for the current scope and contains the top 5 users by spend for the last month + +Date labels: +- daily uses `YYYY-MM-DD` +- weekly uses `YYYY-MM-DD/YYYY-MM-DD` where the label represents week start and week end +- monthly uses `YYYY-MM` + +Examples: +- `dailySpecialDistribution[i]` shows one calendar day from the last month +- `weeklySpecialDistribution[i]` shows one calendar week from the last 5 months +- `monthlyCodioProvidedDistribution[i]` shows one calendar month from the last 5 months +- `topFive[i]` shows one of the five users with the highest spend for the last month in the current org or course scope + +How to read `topFive` + +- `topFive` is calculated for the last month +- it is scoped to the current entity: + - for org statistics, top users inside that organization + - for course statistics, top users inside that course +- it is not split by `codio-special` or `codio-provided`; it reflects total spend in scope for the last month +- it is useful for quickly identifying the most expensive users for investigation, outreach, or manual policy review + +How to read the charts + +Daily chart: +- use it to see short-term volatility and spikes +- `maxUserSpend` highlights strongest single-user bursts in a day +- `medianUserSpend` and `avgUserSpend` show whether broad usage is rising or only a few users are spiking + +Weekly chart: +- use it to see medium-term usage stabilization +- compare weekly `p95UserSpend` and `p99UserSpend` across weeks +- this is useful for tuning weekly guardrails + +Monthly chart: +- use it for budget policy and allowance planning +- monthly changes are less noisy and better reflect stable behavior +- a rising monthly `medianUserSpend` means the typical user is genuinely spending more + +How to use these metrics for limits + +Daily limits: +- look at `daily...Distribution` +- a high `p99UserSpend` with a low median usually means a few strong outliers +- useful for anti-spike or abuse protection + +Weekly limits: +- look at `weekly...Distribution` +- stable weekly `p95UserSpend` can guide soft-limit candidates +- rising weekly `maxUserSpend` may justify a hard cap or alerts + +Monthly limits: +- look at `monthly...Distribution` +- this is the best signal for recurring allowance defaults +- if monthly `medianUserSpend` stays low but `p99UserSpend` rises, the tail is getting heavier without broad adoption + +Practical recommendation flow + +1. Start with monthly series: +- review the trend of `medianUserSpend`, `p95UserSpend`, and `p99UserSpend` +- use this to set or revise monthly allowances + +2. Check weekly series: +- see whether usage changes smoothly week to week or has temporary bursts +- if weekly p95 stays stable but max jumps, use alerts before stricter limits + +3. Check daily series: +- use it for operational safety and anomaly detection +- daily max and p99 are especially useful for identifying runaway prompts or abuse + +Suggested interpretation patterns + +Pattern A: Low median, high max, high p99 only on a few days +- normal usage is cheap +- spikes are rare and sharp +- recommended action: + - keep daily hard caps + - avoid lowering broad monthly limits unnecessarily + +Pattern B: Median and avg both trend upward over weeks and months +- usage is growing across the user base, not just in outliers +- recommended action: + - revise default weekly/monthly budgets upward if product usage is healthy + +Pattern C: Monthly p95 rises while median stays flat +- most users are stable, but heavy-user tail is getting more expensive +- recommended action: + - keep default limits, but strengthen tail controls for power users + +Pattern D: Weekly and monthly max rise together +- expensive usage is not only a single-day anomaly +- recommended action: + - investigate long-running high-cost users and review policy settings + +Limitations of the current model + +- these arrays show metric trends over time, not spend histograms by bucket +- percentiles can be noisy when the number of active users in a period is low +- daily values are naturally more volatile than weekly/monthly values +- recommendations should still be combined with product and business context + +Suggested UI usage + +For each org or course, show: +- a `topFive` table or side panel with: + - `userId` + - `spendLastMonth` +- separate charts for: +- daily special +- weekly special +- monthly special +- daily codio-provided +- weekly codio-provided +- monthly codio-provided + +For each chart, plot one or more lines: +- `medianUserSpend` +- `p95UserSpend` +- `p99UserSpend` +- optionally `maxUserSpend` + +Suggested helper text: +- "Daily p99 shows near-worst-case per-user spend for a single day" +- "Weekly median shows the typical user spend for a full calendar week" +- "Monthly p95 is a strong candidate input for limit policy reviews" + +This makes the dashboard useful for monitoring trend shifts, tuning limits, and spotting anomalous growth. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 7212160..0000000 --- a/TODO.md +++ /dev/null @@ -1,298 +0,0 @@ -{ - "course_id": "course_ai_lit_2026", - "course_name": "AI Literacy Basics", - "org_id": "org_skolkovo_01", - "currency": "USD", - "time_frame": { - "start": "2026-08-01T00:00:00Z", - "end": "2026-08-31T23:59:59Z" - }, - "financial_kpis": { - "total_course_spend": 2450.75, - "max_user_spend": 145.20, - "median_user_spend": 18.50, - "avg_user_spend": 24.02 - }, - "bell_curve_data": [ - { "cost_bucket_usd": "0-5", "user_count": 15 }, - { "cost_bucket_usd": "5-15", "user_count": 45 }, - { "cost_bucket_usd": "15-25", "user_count": 120 }, - { "cost_bucket_usd": "25-40", "user_count": 85 }, - { "cost_bucket_usd": "40-70", "user_count": 30 }, - { "cost_bucket_usd": "70-100", "user_count": 8 }, - { "cost_bucket_usd": "100+", "user_count": 2 } - ], - "users_table": [ - { - "user_id": "user_student_442", - "name": "Иван Иванов", - "total_requests": 340, - "total_spend": 18.45, - "budget_limit_usd": 50.00, - "limit_used_percentage": 36.9 - }, - { - "user_id": "user_student_999", - "name": "Алексей Петров (Аномалия)", - "total_requests": 4500, - "total_spend": 145.20, - "budget_limit_usd": 150.00, - "limit_used_percentage": 96.8 - } - ] -} - ------------------------------------ - -statistics API response notes: -- all period-based distributions and KPIs below are calculated only from data not older than the last 5 months -- `costs.1month` and `costs.5month` remain cumulative spend windows -- period distributions are built from aggregated `user-day`, `user-week`, and `user-month` spend values - -cost: -{ - "1month": float, - "5month": float -} - -costPack: -{ - "codioProvided": cost, - "codioSpecial": cost -} - -bellCurveDataPoint: -{ - "costBucketUsd": string, - "sampleCount": int, - "serialNo": int -} - -financialKpis: -{ - "maxUserSpend": float, - "medianUserSpend": float, - "avgUserSpend": float, - "p90UserSpend": float, - "p95UserSpend": float, - "p99UserSpend": float, - "sampleCount": int, - "recommendedSoftLimitUsd": float, - "recommendedHardLimitUsd": float -} - -spendPeriodStatistics: -{ - "bellCurve": [bellCurveDataPoint], - "financialKpis": financialKpis -} - ------------------------ - -all: -{ - "total": costPack, - "orgs": [ - { - "id": string, - "costs": costPack - } - ] -} - ---------------------- - -org: -{ - "id": string, - "costs": costPack, - "courses": [ - { - "id": string, - "costs": costPack - } - ], - "dailySpecial": spendPeriodStatistics, - "weeklySpecial": spendPeriodStatistics, - "monthlySpecial": spendPeriodStatistics -} - -Recommended buckets for org special: -- daily: 0-1, 1-3, 3-5, 5-10, 10-20, 20+ -- weekly: 0-5, 5-10, 10-20, 20-50, 50-100, 100+ -- monthly: 0-10, 10-25, 25-50, 50-100, 100-250, 250+ - ---------------------- - -course: -{ - "id": string, - "costs": costPack, - "dailyCodioProvided": spendPeriodStatistics, - "weeklyCodioProvided": spendPeriodStatistics, - "monthlyCodioProvided": spendPeriodStatistics -} - -Recommended buckets for course codio-provided: -- daily: 0-1, 1-3, 3-5, 5-10, 10-20, 20+ -- weekly: 0-5, 5-10, 10-20, 20-50, 50-100, 100+ -- monthly: 0-10, 10-25, 25-50, 50-100, 100-250, 250+ - ------------------------------ - -How to interpret the returned statistics - -General principles: -- all `daily*`, `weekly*`, and `monthly*` blocks are calculated only from events from the last 5 months -- each graph is built from aggregated spend per user per period, not from lifetime spend -- the backend first computes: - - one value per `user-day` - - one value per `user-week` - - one value per `user-month` -- then it builds distributions and KPIs from those values - -What `costs` means: -- `costs.codioProvided.1month` — cumulative spend for codio-provided traffic over the last 1 month -- `costs.codioProvided.5month` — cumulative spend for codio-provided traffic over the last 5 months -- `costs.codioSpecial.1month` — cumulative spend for codio-special traffic over the last 1 month -- `costs.codioSpecial.5month` — cumulative spend for codio-special traffic over the last 5 months -- this block is useful for total budget visibility, but by itself is not enough for choosing period-based limits - -What `spendPeriodStatistics` means: -- `bellCurve` shows the distribution of spend samples for a specific period -- `financialKpis` shows summary statistics over the same sample set -- `sampleCount` is the number of aggregated samples used for that period - -Example: -- if `dailySpecial.financialKpis.sampleCount = 1200` -- this means the dataset contains 1200 `user-day` spend values -- not necessarily 1200 unique users -- one active user can contribute multiple daily samples on different days - -How to read the bell curve / histogram - -For org-level `dailySpecial`: -- every sample is one user's total `codio-special` spend within one calendar day -- bucket `0-1` means: number of user-days where spend was less than 1 USD -- bucket `1-3` means: number of user-days where spend was from 1 USD to under 3 USD -- bucket `20+` means: number of user-days where spend was 20 USD or more - -For course-level `weeklyCodioProvided`: -- every sample is one user's total `codio-provided` spend within one calendar week -- if the rightmost buckets are growing, users increasingly hit high weekly cost ranges - -What the graph tells you: -- left-heavy distribution usually means most user-periods are cheap -- a long right tail means there are rare but expensive spikes -- if the center of mass shifts right over time, existing limits may become too strict -- if a large share of samples already sits near your current limit, many normal users may get blocked - -What each KPI means - -For every period block (`daily*`, `weekly*`, `monthly*`): -- `maxUserSpend` — the largest spend observed in one user-period sample -- `medianUserSpend` — the middle spend value; 50% of samples are below it -- `avgUserSpend` — arithmetic average across all samples -- `p90UserSpend` — 90% of samples are at or below this value -- `p95UserSpend` — 95% of samples are at or below this value -- `p99UserSpend` — 99% of samples are at or below this value -- `recommendedSoftLimitUsd` — current recommended soft threshold, equal to `p95` -- `recommendedHardLimitUsd` — current recommended hard threshold, equal to `p99 * 1.2` - -How to use these metrics for limits - -Daily limit: -- use `daily*` blocks -- good starting point: - - soft limit = `p95` - - hard limit = `p99 * 1.2` -- meaning: - - soft limit should catch unusually expensive daily behavior while affecting only a small minority of user-days - - hard limit should catch only strong outliers or suspicious spikes - -Weekly limit: -- use `weekly*` blocks -- useful when daily usage is noisy, but you want to control total weekly burn -- if weekly p95 is stable but daily max is noisy, a weekly limit may be more product-friendly than a strict daily cap - -Monthly limit: -- use `monthly*` blocks -- useful for budget governance and subscription-style controls -- if monthly p95 is close to business budget expectations, it is a strong candidate for the default monthly allowance - -Practical recommendation flow - -1. Start with daily statistics: -- inspect `daily*` bell curve -- check where most samples are concentrated -- compare current or planned daily limit with `p95` and `p99` - -2. Check weekly statistics: -- confirm that users with many moderate daily sessions do not accumulate into unexpectedly high weekly totals -- if weekly tail is too wide, add a weekly limit even if daily looks acceptable - -3. Check monthly statistics: -- make sure the monthly allowance matches your budget model -- monthly limit should reflect normal heavy users, not only median users - -4. Review tails: -- if `maxUserSpend` is much larger than `p99UserSpend`, the system has extreme outliers -- such cases often justify a hard limit, anomaly alert, or additional investigation - -Suggested interpretation patterns - -Pattern A: Most samples in the first buckets, low p95, very high max -- normal usage is cheap -- there are rare spikes or outliers -- recommended action: - - keep soft limit near p95 - - keep hard limit significantly above soft limit - - investigate top offenders separately - -Pattern B: Distribution centered close to the upper buckets -- many users naturally consume a lot -- low limits will likely block legitimate traffic -- recommended action: - - increase default limits - - use weekly/monthly controls instead of aggressive daily limits - -Pattern C: Daily looks healthy, weekly/monthly tails are wide -- individual days are fine, but sustained usage is expensive -- recommended action: - - keep daily limit moderate - - add stronger weekly/monthly guardrails - -Pattern D: p95 and p99 are both moving upward over time -- usage pattern is structurally changing -- recommended action: - - review limits periodically - - consider time-series tracking for p95/p99 as the next dashboard enhancement - -Limitations of the current model - -- the current API returns aggregated distributions and KPIs, not raw per-user samples -- recommendations are heuristic, not policy decisions -- `recommendedSoftLimitUsd` and `recommendedHardLimitUsd` are good defaults, but should still be reviewed against business rules -- if sample count is low, percentiles can be noisy; in that case, org-wide defaults or manual review may be safer - -Suggested UI usage - -For each org or course, show three sections: -- Daily -- Weekly -- Monthly - -Inside each section show: -- histogram from `bellCurve` -- KPI cards: - - median - - p95 - - p99 - - max - - recommended soft limit - - recommended hard limit -- optional helper text: - - "95% of user-periods are below X USD" - - "Only 1% of user-periods exceed Y USD" - -This makes the dashboard directly actionable for tuning quotas and budget limits. diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index 8adce2c..c82e0e7 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -147,12 +147,6 @@ type CostPack struct { CodioSpecial Cost `json:"codioSpecial"` } -type BellCurveDataPoint struct { - CostBucketUsd string `json:"costBucketUsd"` - SampleCount int `json:"sampleCount"` - SerialNo int `json:"serialNo"` -} - type FinancialKpis struct { MaxUserSpend float64 `json:"maxUserSpend"` MedianUserSpend float64 `json:"medianUserSpend"` @@ -165,9 +159,22 @@ type FinancialKpis struct { RecommendedHardLimitUsd float64 `json:"recommendedHardLimitUsd"` } -type SpendPeriodStatistics struct { - BellCurve []BellCurveDataPoint `json:"bellCurve"` - FinancialKpis FinancialKpis `json:"financialKpis"` +type DailySpendDistributionDataPoint struct { + MaxUserSpend float64 `json:"maxUserSpend"` + MedianUserSpend float64 `json:"medianUserSpend"` + AvgUserSpend float64 `json:"avgUserSpend"` + P95UserSpend float64 `json:"p95UserSpend"` + P99UserSpend float64 `json:"p99UserSpend"` + Date string `json:"date"` +} + +type PeriodSpendDistributionDataPoint struct { + MaxUserSpend float64 `json:"maxUserSpend"` + MedianUserSpend float64 `json:"medianUserSpend"` + AvgUserSpend float64 `json:"avgUserSpend"` + P95UserSpend float64 `json:"p95UserSpend"` + P99UserSpend float64 `json:"p99UserSpend"` + DatePeriod string `json:"datePeriod"` } type AllStatisticsData struct { @@ -185,19 +192,32 @@ type ShortCourseStatisticsData struct { Costs CostPack `json:"costs"` } +type TopFiveUserSpend struct { + UserId string `json:"userId"` + SpendLastMonth float64 `json:"spendLastMonth"` +} + type OrgStatisticsData struct { - Id string `json:"id"` - Costs CostPack `json:"costs"` - Courses []ShortCourseStatisticsData `json:"courses"` - DailySpecial SpendPeriodStatistics `json:"dailySpecial"` - WeeklySpecial SpendPeriodStatistics `json:"weeklySpecial"` - MonthlySpecial SpendPeriodStatistics `json:"monthlySpecial"` + Id string `json:"id"` + Costs CostPack `json:"costs"` + Courses []ShortCourseStatisticsData `json:"courses"` + DailySpecialDistribution []DailySpendDistributionDataPoint `json:"dailySpecialDistribution"` + WeeklySpecialDistribution []PeriodSpendDistributionDataPoint `json:"weeklySpecialDistribution"` + MonthlySpecialDistribution []PeriodSpendDistributionDataPoint `json:"monthlySpecialDistribution"` + DailyCodioProvidedDistribution []DailySpendDistributionDataPoint `json:"dailyCodioProvidedDistribution"` + WeeklyCodioProvidedDistribution []PeriodSpendDistributionDataPoint `json:"weeklyCodioProvidedDistribution"` + MonthlyCodioProvidedDistribution []PeriodSpendDistributionDataPoint `json:"monthlyCodioProvidedDistribution"` + TopFive []TopFiveUserSpend `json:"topFive"` } type CourseStatisticsData struct { - Id string `json:"id"` - Costs CostPack `json:"costs"` - DailyCodioProvided SpendPeriodStatistics `json:"dailyCodioProvided"` - WeeklyCodioProvided SpendPeriodStatistics `json:"weeklyCodioProvided"` - MonthlyCodioProvided SpendPeriodStatistics `json:"monthlyCodioProvided"` + Id string `json:"id"` + Costs CostPack `json:"costs"` + DailySpecialDistribution []DailySpendDistributionDataPoint `json:"dailySpecialDistribution"` + WeeklySpecialDistribution []PeriodSpendDistributionDataPoint `json:"weeklySpecialDistribution"` + MonthlySpecialDistribution []PeriodSpendDistributionDataPoint `json:"monthlySpecialDistribution"` + DailyCodioProvidedDistribution []DailySpendDistributionDataPoint `json:"dailyCodioProvidedDistribution"` + WeeklyCodioProvidedDistribution []PeriodSpendDistributionDataPoint `json:"weeklyCodioProvidedDistribution"` + MonthlyCodioProvidedDistribution []PeriodSpendDistributionDataPoint `json:"monthlyCodioProvidedDistribution"` + TopFive []TopFiveUserSpend `json:"topFive"` } diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 78677fe..26e4931 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -6,9 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "math" "slices" - "sort" "strings" "time" @@ -703,26 +701,46 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even return strings.Compare(a.Id, b.Id) }) - dailySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "day") + dailySpecial, err := s.getDailyDistribution([]string{orgTagPrefix + *id}, codioSpecialTag) if err != nil { return nil, err } - weeklySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "week") + weeklySpecial, err := s.getPeriodDistribution([]string{orgTagPrefix + *id}, codioSpecialTag, "week") if err != nil { return nil, err } - monthlySpecial, err := s.getPeriodSpendStatistics([]string{orgTagPrefix + *id}, codioSpecialTag, "month") + monthlySpecial, err := s.getPeriodDistribution([]string{orgTagPrefix + *id}, codioSpecialTag, "month") + if err != nil { + return nil, err + } + dailyProvided, err := s.getDailyDistribution([]string{orgTagPrefix + *id}, codioProvidedTag) + if err != nil { + return nil, err + } + weeklyProvided, err := s.getPeriodDistribution([]string{orgTagPrefix + *id}, codioProvidedTag, "week") + if err != nil { + return nil, err + } + monthlyProvided, err := s.getPeriodDistribution([]string{orgTagPrefix + *id}, codioProvidedTag, "month") + if err != nil { + return nil, err + } + topFive, err := s.getTopFiveUserSpends([]string{orgTagPrefix + *id}) if err != nil { return nil, err } result.OrgStatisticsData = &event.OrgStatisticsData{ - Id: *id, - Costs: *costs, - Courses: courses, - DailySpecial: *dailySpecial, - WeeklySpecial: *weeklySpecial, - MonthlySpecial: *monthlySpecial, + Id: *id, + Costs: *costs, + Courses: courses, + DailySpecialDistribution: dailySpecial, + WeeklySpecialDistribution: weeklySpecial, + MonthlySpecialDistribution: monthlySpecial, + DailyCodioProvidedDistribution: dailyProvided, + WeeklyCodioProvidedDistribution: weeklyProvided, + MonthlyCodioProvidedDistribution: monthlyProvided, + TopFive: topFive, } case event.StisticLevels.Course: if id == nil || len(*id) == 0 { @@ -734,25 +752,45 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even return nil, err } - dailyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "day") + dailySpecial, err := s.getDailyDistribution([]string{courseTagPrefix + *id}, codioSpecialTag) if err != nil { return nil, err } - weeklyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "week") + weeklySpecial, err := s.getPeriodDistribution([]string{courseTagPrefix + *id}, codioSpecialTag, "week") if err != nil { return nil, err } - monthlyProvided, err := s.getPeriodSpendStatistics([]string{courseTagPrefix + *id}, codioProvidedTag, "month") + monthlySpecial, err := s.getPeriodDistribution([]string{courseTagPrefix + *id}, codioSpecialTag, "month") + if err != nil { + return nil, err + } + dailyProvided, err := s.getDailyDistribution([]string{courseTagPrefix + *id}, codioProvidedTag) + if err != nil { + return nil, err + } + weeklyProvided, err := s.getPeriodDistribution([]string{courseTagPrefix + *id}, codioProvidedTag, "week") + if err != nil { + return nil, err + } + monthlyProvided, err := s.getPeriodDistribution([]string{courseTagPrefix + *id}, codioProvidedTag, "month") + if err != nil { + return nil, err + } + topFive, err := s.getTopFiveUserSpends([]string{courseTagPrefix + *id}) if err != nil { return nil, err } result.CourseStatisticsData = &event.CourseStatisticsData{ - Id: *id, - Costs: *costs, - DailyCodioProvided: *dailyProvided, - WeeklyCodioProvided: *weeklyProvided, - MonthlyCodioProvided: *monthlyProvided, + Id: *id, + Costs: *costs, + DailySpecialDistribution: dailySpecial, + WeeklySpecialDistribution: weeklySpecial, + MonthlySpecialDistribution: monthlySpecial, + DailyCodioProvidedDistribution: dailyProvided, + WeeklyCodioProvidedDistribution: weeklyProvided, + MonthlyCodioProvidedDistribution: monthlyProvided, + TopFive: topFive, } default: return nil, internal_errors.NewValidationError("invalid level") @@ -930,32 +968,106 @@ func (s *Store) GetCourseCostPack(courseId string) (*event.CostPack, error) { return s.getTotalCostPack([]string{courseTagPrefix + courseId}) } -func (s *Store) getPeriodSpendStatistics(filterTags []string, costTypeTag, period string) (*event.SpendPeriodStatistics, error) { - values, err := s.getUserPeriodSpendValues(filterTags, costTypeTag, period) +func (s *Store) getDailyDistribution(filterTags []string, costTypeTag string) ([]event.DailySpendDistributionDataPoint, error) { + start := beginningOfDay(time.Now().UTC()).AddDate(0, 0, -29) + end := beginningOfDay(time.Now().UTC()).AddDate(0, 0, 1) + + rows, err := s.getPeriodKpisRows(filterTags, costTypeTag, "day", start, end) if err != nil { return nil, err } - buckets, err := getBucketsForPeriod(period) - if err != nil { - return nil, err + rowMap := make(map[string]financialKpiRow, len(rows)) + for _, row := range rows { + rowMap[row.periodStart.Format("2006-01-02")] = row + } + + result := make([]event.DailySpendDistributionDataPoint, 0, 30) + for current := start; current.Before(end); current = current.AddDate(0, 0, 1) { + key := current.Format("2006-01-02") + row, ok := rowMap[key] + if !ok { + result = append(result, event.DailySpendDistributionDataPoint{Date: key}) + continue + } + result = append(result, event.DailySpendDistributionDataPoint{ + MaxUserSpend: row.max, + MedianUserSpend: row.median, + AvgUserSpend: row.avg, + P95UserSpend: row.p95, + P99UserSpend: row.p99, + Date: key, + }) } - return &event.SpendPeriodStatistics{ - BellCurve: buildBellCurve(values, buckets), - FinancialKpis: *buildFinancialKpis(values), - }, nil + return result, nil } -func (s *Store) getUserPeriodSpendValues(filterTags []string, costTypeTag, period string) ([]float64, error) { - if period != "day" && period != "week" && period != "month" { +func (s *Store) getPeriodDistribution(filterTags []string, costTypeTag, period string) ([]event.PeriodSpendDistributionDataPoint, error) { + var start time.Time + var end time.Time + + now := time.Now().UTC() + switch period { + case "week": + start = beginningOfWeek(now).AddDate(0, 0, -7*20) + end = beginningOfWeek(now).AddDate(0, 0, 7) + case "month": + start = beginningOfMonth(now).AddDate(0, -4, 0) + end = beginningOfMonth(now).AddDate(0, 1, 0) + default: return nil, internal_errors.NewValidationError("unsupported period") } - fiveMonthsAgo := time.Now().Add(statisticsLookbackAge).Unix() - args := []any{userTagPrefix, pq.Array([]string{costTypeTag}), fiveMonthsAgo} - conditions := []string{"tag.tag LIKE $1 || '%'", "e.tags @> $2", "e.created_at >= $3"} - index := 4 + lookbackStart := time.Now().UTC().Add(statisticsLookbackAge) + if start.Before(lookbackStart) { + start = truncateToPeriodStart(lookbackStart, period) + } + + rows, err := s.getPeriodKpisRows(filterTags, costTypeTag, period, start, end) + if err != nil { + return nil, err + } + + rowMap := make(map[string]financialKpiRow, len(rows)) + for _, row := range rows { + rowMap[formatPeriodLabel(row.periodStart, period)] = row + } + + result := []event.PeriodSpendDistributionDataPoint{} + for current := start; current.Before(end); current = addPeriod(current, period) { + label := formatPeriodLabel(current, period) + row, ok := rowMap[label] + if !ok { + result = append(result, event.PeriodSpendDistributionDataPoint{DatePeriod: label}) + continue + } + result = append(result, event.PeriodSpendDistributionDataPoint{ + MaxUserSpend: row.max, + MedianUserSpend: row.median, + AvgUserSpend: row.avg, + P95UserSpend: row.p95, + P99UserSpend: row.p99, + DatePeriod: label, + }) + } + + return result, nil +} + +type financialKpiRow struct { + periodStart time.Time + max float64 + median float64 + avg float64 + p95 float64 + p99 float64 +} + +func (s *Store) getPeriodKpisRows(filterTags []string, costTypeTag, period string, start, end time.Time) ([]financialKpiRow, error) { + args := []any{userTagPrefix, pq.Array([]string{costTypeTag}), start.Unix(), end.Unix()} + conditions := []string{"tag.tag LIKE $1 || '%'", "e.tags @> $2", "e.created_at >= $3", "e.created_at < $4"} + index := 5 for _, tag := range filterTags { conditions = append(conditions, fmt.Sprintf("e.tags @> $%d", index)) @@ -963,132 +1075,155 @@ func (s *Store) getUserPeriodSpendValues(filterTags []string, costTypeTag, perio index++ } + periodExpr := fmt.Sprintf("date_trunc('%s', timezone('UTC', to_timestamp(e.created_at)))", period) query := fmt.Sprintf(` - SELECT user_period_cost - FROM ( + WITH per_user_period AS ( SELECT + %s AS period_start, regexp_replace(tag.tag, '^' || $1, '') AS user_id, - date_trunc('%s', to_timestamp(e.created_at)) AS period_start, COALESCE(SUM(e.cost_in_usd), 0) AS user_period_cost FROM events e, LATERAL unnest(e.tags) AS tag(tag) WHERE %s - GROUP BY regexp_replace(tag.tag, '^' || $1, ''), date_trunc('%s', to_timestamp(e.created_at)) - ) AS aggregated_user_spend - ORDER BY period_start, user_id - `, period, strings.Join(conditions, " AND "), period) + GROUP BY %s, regexp_replace(tag.tag, '^' || $1, '') + ) + SELECT + period_start, + COALESCE(MAX(user_period_cost), 0) AS max_user_spend, + COALESCE(percentile_cont(0.5) WITHIN GROUP (ORDER BY user_period_cost), 0) AS median_user_spend, + COALESCE(AVG(user_period_cost), 0) AS avg_user_spend, + COALESCE(percentile_cont(0.95) WITHIN GROUP (ORDER BY user_period_cost), 0) AS p95_user_spend, + COALESCE(percentile_cont(0.99) WITHIN GROUP (ORDER BY user_period_cost), 0) AS p99_user_spend + FROM per_user_period + GROUP BY period_start + ORDER BY period_start + `, periodExpr, strings.Join(conditions, " AND "), periodExpr) ctx, cancel := context.WithTimeout(context.Background(), s.rt) defer cancel() - rows, err := s.db.QueryContext(ctx, query, args...) + queryRows, err := s.db.QueryContext(ctx, query, args...) if err != nil { return nil, err } - defer rows.Close() + defer queryRows.Close() - values := []float64{} - for rows.Next() { - var spend float64 - if err := rows.Scan(&spend); err != nil { + result := []financialKpiRow{} + for queryRows.Next() { + var row financialKpiRow + if err := queryRows.Scan(&row.periodStart, &row.max, &row.median, &row.avg, &row.p95, &row.p99); err != nil { return nil, err } - values = append(values, spend) + result = append(result, row) } - if err := rows.Err(); err != nil { + if err := queryRows.Err(); err != nil { return nil, err } - return values, nil + return result, nil } -type costBucket struct { - label string - max float64 -} +func (s *Store) getTopFiveUserSpends(filterTags []string) ([]event.TopFiveUserSpend, error) { + start := beginningOfMonth(time.Now().UTC()) + end := beginningOfMonth(time.Now().UTC()).AddDate(0, 1, 0) -func getBucketsForPeriod(period string) ([]costBucket, error) { - switch period { - case "day": - return []costBucket{{label: "0-1", max: 1}, {label: "1-3", max: 3}, {label: "3-5", max: 5}, {label: "5-10", max: 10}, {label: "10-20", max: 20}, {label: "20+", max: math.Inf(1)}}, nil - case "week": - return []costBucket{{label: "0-5", max: 5}, {label: "5-10", max: 10}, {label: "10-20", max: 20}, {label: "20-50", max: 50}, {label: "50-100", max: 100}, {label: "100+", max: math.Inf(1)}}, nil - case "month": - return []costBucket{{label: "0-10", max: 10}, {label: "10-25", max: 25}, {label: "25-50", max: 50}, {label: "50-100", max: 100}, {label: "100-250", max: 250}, {label: "250+", max: math.Inf(1)}}, nil - default: - return nil, internal_errors.NewValidationError("unsupported period") + args := []any{userTagPrefix, start.Unix(), end.Unix()} + conditions := []string{"tag.tag LIKE $1 || '%'", "e.created_at >= $2", "e.created_at < $3"} + index := 4 + + for _, tag := range filterTags { + conditions = append(conditions, fmt.Sprintf("e.tags @> $%d", index)) + args = append(args, pq.Array([]string{tag})) + index++ } -} -func buildBellCurve(values []float64, buckets []costBucket) []event.BellCurveDataPoint { - counts := make([]int, len(buckets)) - for _, v := range values { - for i, bucket := range buckets { - if v < bucket.max { - counts[i]++ - break - } - } + query := fmt.Sprintf(` + SELECT + regexp_replace(tag.tag, '^' || $1, '') AS user_id, + COALESCE(SUM(e.cost_in_usd), 0) AS spend_last_month + FROM events e, + LATERAL unnest(e.tags) AS tag(tag) + WHERE %s + GROUP BY regexp_replace(tag.tag, '^' || $1, '') + ORDER BY spend_last_month DESC, user_id ASC + LIMIT 5 + `, strings.Join(conditions, " AND ")) + + ctx, cancel := context.WithTimeout(context.Background(), s.rt) + defer cancel() + + rows, err := s.db.QueryContext(ctx, query, args...) + if err != nil { + return nil, err } + defer rows.Close() - result := make([]event.BellCurveDataPoint, 0, len(buckets)) - for i, bucket := range buckets { - result = append(result, event.BellCurveDataPoint{ - CostBucketUsd: bucket.label, - SampleCount: counts[i], - SerialNo: i + 1, - }) + result := []event.TopFiveUserSpend{} + for rows.Next() { + var row event.TopFiveUserSpend + if err := rows.Scan(&row.UserId, &row.SpendLastMonth); err != nil { + return nil, err + } + result = append(result, row) + } + if err := rows.Err(); err != nil { + return nil, err } - return result + return result, nil } -func buildFinancialKpis(values []float64) *event.FinancialKpis { - if len(values) == 0 { - return &event.FinancialKpis{} - } - - sorted := append([]float64(nil), values...) - sort.Float64s(sorted) +func beginningOfDay(t time.Time) time.Time { + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) +} - total := 0.0 - for _, v := range sorted { - total += v +func beginningOfWeek(t time.Time) time.Time { + dayStart := beginningOfDay(t) + weekday := int(dayStart.Weekday()) + if weekday == 0 { + weekday = 7 } + return dayStart.AddDate(0, 0, -(weekday - 1)) +} - p95 := percentile(sorted, 0.95) - p99 := percentile(sorted, 0.99) +func beginningOfMonth(t time.Time) time.Time { + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) +} - return &event.FinancialKpis{ - MaxUserSpend: sorted[len(sorted)-1], - MedianUserSpend: percentile(sorted, 0.50), - AvgUserSpend: total / float64(len(sorted)), - P90UserSpend: percentile(sorted, 0.90), - P95UserSpend: p95, - P99UserSpend: p99, - SampleCount: len(sorted), - RecommendedSoftLimitUsd: p95, - RecommendedHardLimitUsd: p99 * 1.2, +func truncateToPeriodStart(t time.Time, period string) time.Time { + switch period { + case "day": + return beginningOfDay(t) + case "week": + return beginningOfWeek(t) + case "month": + return beginningOfMonth(t) + default: + return beginningOfDay(t) } } -func percentile(sorted []float64, p float64) float64 { - if len(sorted) == 0 { - return 0 - } - if len(sorted) == 1 { - return sorted[0] +func addPeriod(t time.Time, period string) time.Time { + switch period { + case "week": + return t.AddDate(0, 0, 7) + case "month": + return t.AddDate(0, 1, 0) + default: + return t.AddDate(0, 0, 1) } +} - position := p * float64(len(sorted)-1) - lower := int(math.Floor(position)) - upper := int(math.Ceil(position)) - if lower == upper { - return sorted[lower] +func formatPeriodLabel(start time.Time, period string) string { + switch period { + case "week": + end := start.AddDate(0, 0, 6) + return fmt.Sprintf("%s/%s", start.Format("2006-01-02"), end.Format("2006-01-02")) + case "month": + return start.Format("2006-01") + default: + return start.Format("2006-01-02") } - - weight := position - float64(lower) - return sorted[lower] + (sorted[upper]-sorted[lower])*weight } func (s *Store) GetAggregatedEventByDayDataPoints(start, end int64, keyIds []string) ([]*event.DataPointV2, error) { From 9b7be7bef5ae1b7682c3fba236145f8d64f7dc6d Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 9 Sep 2026 12:57:00 +0100 Subject: [PATCH 07/14] clean --- internal/storage/postgresql/event.go | 45 ---------------------------- 1 file changed, 45 deletions(-) diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 26e4931..7c8b14f 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -799,51 +799,6 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even return result, nil } -// { -// "total": costPack, -// "orgs": [ -// { -// "id": string, -// "costs": costPack, -// }, -// ] -// } -// -// val ORG_TAG_PREFIX = "org-tag-" -// val USER_TAG_PREFIX = "user-tag-" -// val COURSE_TAG_PREFIX = "course-tag-" -// -// val CODIO_SPECIAL_TAG = "codio-special" -// val CODIO_PROVIDED_TAG = "codio-provided" -// -// type Cost struct { -// OneMonth float64 `json:"1month"` -// FiveMonth float64 `json:"5month"` -// } - -// type CostPack struct { -// CodioProvided Cost `json:"codioProvided"` -// CodioSpecial Cost `json:"codioSpecial"` -// } -// -// SELECT -// substring(elem FROM 'org-tag-(.+)') AS user_id, -// count(*) AS total_records -// FROM -// your_table, -// unnest(your_array_column) AS elem -// WHERE -// elem LIKE 'org-tag--%' -// GROUP BY -// user_id; -// -// -// - -// "event_id" "created_at" "tags" "key_id" "cost_in_usd" "provider" "model" "status_code" "prompt_token_count" "completion_token_count" "latency_in_ms" "path" "method" "custom_id" "request" "response" "user_id" "action" "policy_id" "route_id" "correlation_id" "metadata" -// "0df9bc37-0e76-4192-ad43-d721acf6d638" 1785927977 "{org-tag-134db24b-62c3-44f5-b929-ec668f13d98b,user-tag-00112233-4455-6677-9cef-5b5dd134bfbf,course-tag-44e1ee81b625dbb97e4b5f2c57ab3183,codio-special}" "a950fce4-f91a-4195-9202-24480956d2f7" 0 "openai" "gpt-5.4-nano" 401 0 0 266 "/api/providers/openai/v1/responses" "POST" "{""input"": [{""role"": ""user"", ""content"": [{""text"": ""hello"", ""type"": ""input_text""}]}], ""model"": ""gpt-5.4-nano"", ""stream"": true}" "{}" "6a797d0f-5480-4d8c-bf7a-931f97e00666" "{}" -// - const ( orgTagPrefix = "org-tag-" userTagPrefix = "user-tag-" From 7815c829dc77742ad74e65b039158fc90e262581 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Wed, 9 Sep 2026 15:21:54 +0100 Subject: [PATCH 08/14] format --- internal/storage/postgresql/event.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 7c8b14f..86cc459 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -934,12 +934,12 @@ func (s *Store) getDailyDistribution(filterTags []string, costTypeTag string) ([ rowMap := make(map[string]financialKpiRow, len(rows)) for _, row := range rows { - rowMap[row.periodStart.Format("2006-01-02")] = row + rowMap[row.periodStart.Format("Jan-02")] = row } result := make([]event.DailySpendDistributionDataPoint, 0, 30) for current := start; current.Before(end); current = current.AddDate(0, 0, 1) { - key := current.Format("2006-01-02") + key := current.Format("Jan-02") row, ok := rowMap[key] if !ok { result = append(result, event.DailySpendDistributionDataPoint{Date: key}) @@ -1173,11 +1173,11 @@ func formatPeriodLabel(start time.Time, period string) string { switch period { case "week": end := start.AddDate(0, 0, 6) - return fmt.Sprintf("%s/%s", start.Format("2006-01-02"), end.Format("2006-01-02")) + return fmt.Sprintf("%s/%s", start.Format("Jan-02"), end.Format("Jan-02")) case "month": - return start.Format("2006-01") + return start.Format("Jan") default: - return start.Format("2006-01-02") + return start.Format("Jan-02") } } From d1ff5cd7be943841fab6d0e97c0915353c1fa9f3 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Thu, 10 Sep 2026 18:37:32 +0100 Subject: [PATCH 09/14] cache --- cmd/bricksllm/main.go | 11 ++- internal/event/key_reporting.go | 7 ++ internal/manager/reporting.go | 58 +++++++++-- internal/server/web/admin/reporting.go | 8 ++ internal/storage/redis/statistic-cache.go | 112 ++++++++++++++++++++++ 5 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 internal/storage/redis/statistic-cache.go diff --git a/cmd/bricksllm/main.go b/cmd/bricksllm/main.go index 40913bc..009dfce 100644 --- a/cmd/bricksllm/main.go +++ b/cmd/bricksllm/main.go @@ -297,6 +297,14 @@ func main() { log.Sugar().Fatalf("error connecting to secondary keys redis storage: %v", err) } + statisticsRedisCache := redis.NewClient(defaultRedisOption(cfg, 13)) + + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err = statisticsRedisCache.Ping(ctx).Err(); err != nil { + log.Sugar().Fatalf("error connecting to statistics redis storage: %v", err) + } + rateLimitCache := redisStorage.NewCache(rateLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) costLimitCache := redisStorage.NewCache(costLimitRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) costStorage := redisStorage.NewStore(costRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) @@ -312,6 +320,7 @@ func main() { keysCache := redisStorage.NewKeysCache(keysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) secondaryKeysCache := redisStorage.NewSecondaryKeysCache(secondaryKeysRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) requestsLimitStorage := redisStorage.NewStore(requestsLimitRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) + statisticsCache := redisStorage.NewStatisticCache(statisticsRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) encryptor, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience) if cfg.EnableEncrytion && err != nil { @@ -320,7 +329,7 @@ func main() { v := validator.NewValidator(costLimitCache, rateLimitCache, costStorage, requestsLimitStorage) m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache, secondaryKeysCache, requestsLimitStorage) - krm := manager.NewReportingManager(costStorage, store, store, v) + krm := manager.NewReportingManager(costStorage, store, store, v, statisticsCache) psm := manager.NewProviderSettingsManager(store, psCache, encryptor) cpm := manager.NewCustomProvidersManager(store, cpMemStore) rm := manager.NewRouteManager(store, store, rMemStore, psm) diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index c82e0e7..b976177 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -86,6 +86,13 @@ type StatisticsRequest struct { Id *string `json:"id"` } +func (r *StatisticsRequest) GetCacheKey() string { + if r.Id != nil { + return r.Level + ":" + *r.Id + } + return r.Level +} + type StatisticLevel string var StisticLevels = struct { diff --git a/internal/manager/reporting.go b/internal/manager/reporting.go index 596c42a..ac55d7e 100644 --- a/internal/manager/reporting.go +++ b/internal/manager/reporting.go @@ -2,6 +2,7 @@ package manager import ( "strings" + "time" internal_errors "github.com/bricks-cloud/bricksllm/internal/errors" "github.com/bricks-cloud/bricksllm/internal/event" @@ -21,6 +22,14 @@ type keyValidator interface { Validate(k *key.ResponseKey, promptCost float64) error } +type StatisticsCache interface { + Get(key string) (*event.StatisticsData, error) + Set(key string, val *event.StatisticsData, ttl time.Duration) error + SetInProgress(key string) error + DeleteInProgress(key string) error + IsInProgress(key string) bool +} + type eventStorage interface { GetEvents(userId, customId string, keyIds []string, start, end int64) ([]*event.Event, error) GetEventsV2(req *event.EventRequest) (*event.EventResponse, error) @@ -41,14 +50,16 @@ type ReportingManager struct { cs costStorage ks keyStorage kv keyValidator + sc StatisticsCache } -func NewReportingManager(cs costStorage, ks keyStorage, es eventStorage, kv keyValidator) *ReportingManager { +func NewReportingManager(cs costStorage, ks keyStorage, es eventStorage, kv keyValidator, sc StatisticsCache) *ReportingManager { return &ReportingManager{ cs: cs, ks: ks, es: es, kv: kv, + sc: sc, } } @@ -197,14 +208,49 @@ func (rm *ReportingManager) GetStatistic(r *event.StatisticsRequest) (*event.Sta return nil, err } - statisticsData, err := rm.es.GetStatisticsData(r.GetLevel(), r.Id) + cacheKey := r.GetCacheKey() + if data, err := rm.sc.Get(cacheKey); err == nil { + return &event.StatisticsResponse{ + StatisticsData: data, + }, nil + } + + go rm.backgroundCollectStatisticsData(cacheKey, r.GetLevel(), r.Id) + + timeout := time.After(10 * time.Second) + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-timeout: + return nil, internal_errors.NewNotFoundError("statistics data is not ready yet, please try again later") + case <-ticker.C: + if data, err := rm.sc.Get(cacheKey); err == nil { + return &event.StatisticsResponse{ + StatisticsData: data, + }, nil + } + } + } +} + +func (rm *ReportingManager) backgroundCollectStatisticsData(cacheKey string, level event.StatisticLevel, id *string) { + if rm.sc.IsInProgress(cacheKey) { + return + } + err := rm.sc.SetInProgress(cacheKey) if err != nil { - return nil, err + return } + defer rm.sc.DeleteInProgress(cacheKey) - return &event.StatisticsResponse{ - StatisticsData: statisticsData, - }, nil + statisticsData, err := rm.es.GetStatisticsData(level, id) + + if err != nil { + return + } + _ = rm.sc.Set(cacheKey, statisticsData, time.Hour*24) } func (rm *ReportingManager) GetCustomIds(keyId string) ([]string, error) { diff --git a/internal/server/web/admin/reporting.go b/internal/server/web/admin/reporting.go index 1a07df7..bec502e 100644 --- a/internal/server/web/admin/reporting.go +++ b/internal/server/web/admin/reporting.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "github.com/bricks-cloud/bricksllm/internal/errors" "github.com/bricks-cloud/bricksllm/internal/event" "github.com/bricks-cloud/bricksllm/internal/telemetry" "github.com/bricks-cloud/bricksllm/internal/util" @@ -607,6 +608,13 @@ func getGetStatisticHandler(m KeyReportingManager, prod bool) gin.HandlerFunc { telemetry.Incr("bricksllm.admin.get_get_statistic_handler.get_statistic", nil, 1) logError(log, "error when getting top key ring reporting", prod, err) + + if _, ok := err.(*errors.NotFoundError); ok { + fmt.Println("NotFoundError:", err.Error()) + c.JSON(http.StatusAccepted, &gin.H{"status": "in_progress", "message": "statistics data is being collected, please try again later"}) + return + } + c.JSON(http.StatusInternalServerError, &ErrorResponse{ Type: "/errors/event-reporting-manager", Title: "usage reporting error", diff --git a/internal/storage/redis/statistic-cache.go b/internal/storage/redis/statistic-cache.go new file mode 100644 index 0000000..e692634 --- /dev/null +++ b/internal/storage/redis/statistic-cache.go @@ -0,0 +1,112 @@ +package redis + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/bricks-cloud/bricksllm/internal/event" + "github.com/redis/go-redis/v9" +) + +const inProgressKeyPrefix = "statistic_in_progress_" + +type StatisticCache struct { + client *redis.Client + wt time.Duration + rt time.Duration +} + +func NewStatisticCache(c *redis.Client, wt time.Duration, rt time.Duration) *StatisticCache { + return &StatisticCache{ + client: c, + wt: wt, + rt: rt, + } +} + +func (c *StatisticCache) Set(key string, value *event.StatisticsData, ttl time.Duration) error { + if value == nil { + return errors.New("statistic data to set is nil") + } + ctx, cancel := context.WithTimeout(context.Background(), c.wt) + defer cancel() + bs, err := json.Marshal(value) + if err != nil { + return err + } + err = c.client.Set(ctx, key, bs, ttl).Err() + if err != nil { + return err + } + + return nil +} + +func (c *StatisticCache) Delete(key string) error { + ctx, cancel := context.WithTimeout(context.Background(), c.wt) + defer cancel() + err := c.client.Del(ctx, key).Err() + if err != nil { + return err + } + + return nil +} + +func (c *StatisticCache) Get(key string) (*event.StatisticsData, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.rt) + defer cancel() + + result := c.client.Get(ctx, key) + err := result.Err() + if err != nil { + return nil, err + } + + bs, err := result.Bytes() + if err != nil { + return nil, err + } + + var stat event.StatisticsData + err = json.Unmarshal(bs, &stat) + if err != nil { + return nil, err + } + + return &stat, nil +} + +func (c *StatisticCache) SetInProgress(key string) error { + k := inProgressKeyPrefix + key + ctx, cancel := context.WithTimeout(context.Background(), c.wt) + defer cancel() + err := c.client.Set(ctx, k, true, time.Minute*10).Err() + if err != nil { + return err + } + return nil +} + +func (c *StatisticCache) IsInProgress(key string) bool { + k := inProgressKeyPrefix + key + ctx, cancel := context.WithTimeout(context.Background(), c.rt) + defer cancel() + + result := c.client.Get(ctx, k) + + return !errors.Is(result.Err(), redis.Nil) +} + +func (c *StatisticCache) DeleteInProgress(key string) error { + k := inProgressKeyPrefix + key + ctx, cancel := context.WithTimeout(context.Background(), c.wt) + defer cancel() + err := c.client.Del(ctx, k).Err() + if err != nil { + return err + } + return nil +} From ddb7c478115b5d3cecdf98be0386b3bf18d2113a Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 11 Sep 2026 11:56:11 +0100 Subject: [PATCH 10/14] review --- internal/event/key_reporting.go | 17 +- internal/server/web/admin/reporting.go | 6 +- internal/storage/postgresql/event.go | 19 +- internal/storage/redis/statistic-cache.go | 5 +- mock-data/README.md | 137 --------- mock-data/generate_statistics_mock.go | 346 ---------------------- 6 files changed, 26 insertions(+), 504 deletions(-) delete mode 100644 mock-data/README.md delete mode 100644 mock-data/generate_statistics_mock.go diff --git a/internal/event/key_reporting.go b/internal/event/key_reporting.go index b976177..7aa9c67 100644 --- a/internal/event/key_reporting.go +++ b/internal/event/key_reporting.go @@ -1,6 +1,8 @@ package event import ( + "strings" + internalErrors "github.com/bricks-cloud/bricksllm/internal/errors" ) @@ -87,6 +89,9 @@ type StatisticsRequest struct { } func (r *StatisticsRequest) GetCacheKey() string { + if r.Level == "all" { + return r.Level + } if r.Id != nil { return r.Level + ":" + *r.Id } @@ -95,7 +100,7 @@ func (r *StatisticsRequest) GetCacheKey() string { type StatisticLevel string -var StisticLevels = struct { +var StaticLevels = struct { Unknown StatisticLevel All StatisticLevel Org StatisticLevel @@ -110,13 +115,13 @@ var StisticLevels = struct { func StatisticLevelFromStr(s string) StatisticLevel { switch s { case "all": - return StisticLevels.All + return StaticLevels.All case "org": - return StisticLevels.Org + return StaticLevels.Org case "course": - return StisticLevels.Course + return StaticLevels.Course default: - return StisticLevels.Unknown + return StaticLevels.Unknown } } @@ -124,7 +129,7 @@ func (r *StatisticsRequest) Validate() error { if r.Level != "all" && r.Level != "org" && r.Level != "course" { return internalErrors.NewValidationError("level must be one of 'all', 'org', or 'course'") } - if (r.Level == "org" || r.Level == "course") && r.Id == nil { + if (r.Level == "org" || r.Level == "course") && (r.Id == nil || strings.TrimSpace(*r.Id) == "") { return internalErrors.NewValidationError("id must be provided when level is 'org' or 'course'") } return nil diff --git a/internal/server/web/admin/reporting.go b/internal/server/web/admin/reporting.go index bec502e..1ed884c 100644 --- a/internal/server/web/admin/reporting.go +++ b/internal/server/web/admin/reporting.go @@ -578,7 +578,7 @@ func getGetStatisticHandler(m KeyReportingManager, prod bool) gin.HandlerFunc { data, err := io.ReadAll(c.Request.Body) if err != nil { - logError(log, "error when reading usage reporting request body", prod, err) + logError(log, "error when reading statistics request body", prod, err) c.JSON(http.StatusInternalServerError, &ErrorResponse{ Type: "/errors/request-body-read", Title: "request body reader error", @@ -607,7 +607,7 @@ func getGetStatisticHandler(m KeyReportingManager, prod bool) gin.HandlerFunc { if err != nil { telemetry.Incr("bricksllm.admin.get_get_statistic_handler.get_statistic", nil, 1) - logError(log, "error when getting top key ring reporting", prod, err) + logError(log, "error when getting statistics", prod, err) if _, ok := err.(*errors.NotFoundError); ok { fmt.Println("NotFoundError:", err.Error()) @@ -617,7 +617,7 @@ func getGetStatisticHandler(m KeyReportingManager, prod bool) gin.HandlerFunc { c.JSON(http.StatusInternalServerError, &ErrorResponse{ Type: "/errors/event-reporting-manager", - Title: "usage reporting error", + Title: "statistics reporting error", Status: http.StatusInternalServerError, Detail: err.Error(), Instance: path, diff --git a/internal/storage/postgresql/event.go b/internal/storage/postgresql/event.go index 86cc459..8a37bcc 100644 --- a/internal/storage/postgresql/event.go +++ b/internal/storage/postgresql/event.go @@ -655,7 +655,7 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even result := &event.StatisticsData{} switch level { - case event.StisticLevels.All: + case event.StaticLevels.All: total, err := s.getTotalCostPack(nil) if err != nil { return nil, err @@ -678,7 +678,7 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even Total: *total, Orgs: orgs, } - case event.StisticLevels.Org: + case event.StaticLevels.Org: if id == nil || len(*id) == 0 { return nil, internal_errors.NewValidationError("id must be provided when level is 'org'") } @@ -742,7 +742,7 @@ func (s *Store) GetStatisticsData(level event.StatisticLevel, id *string) (*even MonthlyCodioProvidedDistribution: monthlyProvided, TopFive: topFive, } - case event.StisticLevels.Course: + case event.StaticLevels.Course: if id == nil || len(*id) == 0 { return nil, internal_errors.NewValidationError("id must be provided when level is 'course'") } @@ -851,18 +851,21 @@ func (s *Store) getGroupedCostPack(groupBy string, filterTags []string) (map[str var id string var pack event.CostPack - if err := rows.Scan( + if err2 := rows.Scan( &id, &pack.CodioProvided.OneMonth, &pack.CodioProvided.FiveMonth, &pack.CodioSpecial.OneMonth, &pack.CodioSpecial.FiveMonth, - ); err != nil { - return nil, err + ); err2 != nil { + return nil, err2 } result[id] = pack } + if err2 := rows.Err(); err2 != nil { + return nil, err2 + } return result, nil } @@ -1079,8 +1082,8 @@ func (s *Store) getPeriodKpisRows(filterTags []string, costTypeTag, period strin } func (s *Store) getTopFiveUserSpends(filterTags []string) ([]event.TopFiveUserSpend, error) { - start := beginningOfMonth(time.Now().UTC()) - end := beginningOfMonth(time.Now().UTC()).AddDate(0, 1, 0) + end := time.Now().UTC() + start := end.Add(-30 * 24 * time.Hour) args := []any{userTagPrefix, start.Unix(), end.Unix()} conditions := []string{"tag.tag LIKE $1 || '%'", "e.created_at >= $2", "e.created_at < $3"} diff --git a/internal/storage/redis/statistic-cache.go b/internal/storage/redis/statistic-cache.go index e692634..73926a8 100644 --- a/internal/storage/redis/statistic-cache.go +++ b/internal/storage/redis/statistic-cache.go @@ -94,10 +94,7 @@ func (c *StatisticCache) IsInProgress(key string) bool { k := inProgressKeyPrefix + key ctx, cancel := context.WithTimeout(context.Background(), c.rt) defer cancel() - - result := c.client.Get(ctx, k) - - return !errors.Is(result.Err(), redis.Nil) + return c.client.Get(ctx, k).Err() == nil } func (c *StatisticCache) DeleteInProgress(key string) error { diff --git a/mock-data/README.md b/mock-data/README.md deleted file mode 100644 index 521bb12..0000000 --- a/mock-data/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# Mock data for statistics testing - -This directory contains a Go script for generating synthetic `events` rows in PostgreSQL for testing statistics endpoints and limit-tuning dashboards. - -## Files - -- `generate_statistics_mock.go` — inserts mock rows into the `events` table - -## What the generator creates - -The script generates realistic event traffic across: -- organizations -- courses -- users -- `codio-provided` and `codio-special` usage types - -Each inserted event includes these tags: -- `org-tag-...` -- `course-tag-...` -- `user-tag-...` -- `codio-provided` or `codio-special` -- `mock-generated` - -The `mock-generated` tag is added specifically so the data can be removed manually after testing. - -## Default connection - -By default the script connects to: - -```text -postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432 -``` - -This matches the local PostgreSQL config from `docker-compose.yml`. - -## Run - -From the project root: - -```bash -go run ./mock-data/generate_statistics_mock.go -``` - -Example with explicit parameters: - -```bash -go run ./mock-data/generate_statistics_mock.go \ - -dsn "postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432" \ - -orgs 3 \ - -courses-per-org 4 \ - -users-per-course 80 \ - -days 150 \ - -seed 42 \ - -verbose -``` - -## Flags - -- `-dsn` — PostgreSQL DSN -- `-orgs` — number of organizations to generate -- `-courses-per-org` — number of courses per organization -- `-users-per-course` — number of users per course -- `-days` — how many past days of events to generate -- `-seed` — random seed for reproducible output -- `-verbose` — print periodic progress logs - -## Important note about 5-month statistics window - -Yes — you can intentionally generate data for a period **longer than 5 months** to verify that the statistics code correctly excludes old data. - -For example: - -```bash -go run ./mock-data/generate_statistics_mock.go -days 240 -``` - -This will insert mock events going back roughly 8 months. - -The current statistics implementation: -- uses only the **last 5 months** for: - - `daily*` - - `weekly*` - - `monthly*` - - bell-curve distributions - - percentile-based KPIs and limit recommendations -- also uses the same 5-month window for `costs.*.5month` -- uses the last 1 month for `costs.*.1month` - -So generating `-days` greater than `150` is a valid and useful test scenario. - -## Recommended validation scenarios - -### 1. Normal dashboard test -Generate exactly the recent window: - -```bash -go run ./mock-data/generate_statistics_mock.go -days 150 -``` - -Use this to inspect typical dashboard output. - -### 2. Old-data exclusion test -Generate a wider history: - -```bash -go run ./mock-data/generate_statistics_mock.go -days 240 -``` - -Use this to verify that: -- daily / weekly / monthly statistics do not drift because of older records -- `5month` cost values ignore the oldest generated rows -- recommendations are based only on recent behavior - -### 3. Higher-volume stress test - -```bash -go run ./mock-data/generate_statistics_mock.go -orgs 8 -courses-per-org 10 -users-per-course 200 -days 180 -``` - -Use this to test: -- query performance -- histogram stability -- UI rendering under larger sample counts - -## Cleanup - -To remove only rows generated by this script: - -```sql -DELETE FROM events WHERE tags @> ARRAY['mock-generated']::varchar[]; -``` - -## Notes - -- The generator creates outliers on purpose so that `p95`, `p99`, `maxUserSpend`, and recommended soft/hard limits are meaningful. -- The script is intended for local/dev environments. -- If the `events` table does not exist yet, the script creates it and applies the table alteration used by the application. diff --git a/mock-data/generate_statistics_mock.go b/mock-data/generate_statistics_mock.go deleted file mode 100644 index 65f8f2f..0000000 --- a/mock-data/generate_statistics_mock.go +++ /dev/null @@ -1,346 +0,0 @@ -package main - -import ( - "database/sql" - "flag" - "fmt" - "math/rand" - "strings" - "time" - - "github.com/google/uuid" - "github.com/lib/pq" - - "github.com/bricks-cloud/bricksllm/internal/storage/postgresql" -) - -const ( - mockGeneratedTag = "mock-generated" - orgTagPrefix = "org-tag-" - courseTagPrefix = "course-tag-" - userTagPrefix = "user-tag-" - codioSpecialTag = "codio-special" - codioProvidedTag = "codio-provided" -) - -type config struct { - dsn string - orgCount int - coursesPerOrg int - usersPerCourse int - days int - seed int64 - verbose bool -} - -type courseDef struct { - orgID string - courseID string - users []string -} - -func main() { - cfg := parseFlags() - - store, err := postgresql.NewStore(cfg.dsn, 30*time.Second, 30*time.Second) - if err != nil { - panic(err) - } - - if err := store.CreateEventsTable(); err != nil { - panic(err) - } - if err := store.AlterEventsTable(); err != nil { - panic(err) - } - - db, err := sql.Open("postgres", cfg.dsn) - if err != nil { - panic(err) - } - defer db.Close() - - rng := rand.New(rand.NewSource(cfg.seed)) - courses := buildTopology(cfg) - - inserted, err := seedEvents(db, rng, cfg, courses) - if err != nil { - panic(err) - } - - fmt.Printf("Inserted %d mock events into events table.\n", inserted) - fmt.Printf("All generated rows include tag %q.\n", mockGeneratedTag) - fmt.Printf("Cleanup SQL: DELETE FROM events WHERE tags @> ARRAY['%s']::varchar[];\n", mockGeneratedTag) -} - -func parseFlags() config { - cfg := config{} - flag.StringVar(&cfg.dsn, "dsn", "postgresql:///?sslmode=disable&user=postgres&password=postgres&host=localhost&port=5432", "PostgreSQL DSN") - flag.IntVar(&cfg.orgCount, "orgs", 3, "Number of organizations") - flag.IntVar(&cfg.coursesPerOrg, "courses-per-org", 4, "Number of courses per organization") - flag.IntVar(&cfg.usersPerCourse, "users-per-course", 80, "Number of users per course") - flag.IntVar(&cfg.days, "days", 150, "How many past days of events to generate") - flag.Int64Var(&cfg.seed, "seed", 42, "Random seed for reproducible data") - flag.BoolVar(&cfg.verbose, "verbose", false, "Print progress details") - flag.Parse() - - if cfg.orgCount <= 0 || cfg.coursesPerOrg <= 0 || cfg.usersPerCourse <= 0 || cfg.days <= 0 { - panic("orgs, courses-per-org, users-per-course and days must be positive") - } - - return cfg -} - -func buildTopology(cfg config) []courseDef { - courses := make([]courseDef, 0, cfg.orgCount*cfg.coursesPerOrg) - for orgIdx := 1; orgIdx <= cfg.orgCount; orgIdx++ { - orgID := fmt.Sprintf("mock-org-%02d", orgIdx) - for courseIdx := 1; courseIdx <= cfg.coursesPerOrg; courseIdx++ { - courseID := fmt.Sprintf("mock-course-%02d-%02d", orgIdx, courseIdx) - users := make([]string, 0, cfg.usersPerCourse) - for userIdx := 1; userIdx <= cfg.usersPerCourse; userIdx++ { - users = append(users, fmt.Sprintf("mock-user-%02d-%02d-%03d", orgIdx, courseIdx, userIdx)) - } - courses = append(courses, courseDef{orgID: orgID, courseID: courseID, users: users}) - } - } - return courses -} - -func seedEvents(db *sql.DB, rng *rand.Rand, cfg config, courses []courseDef) (int, error) { - const insertQuery = ` - INSERT INTO events ( - event_id, - created_at, - tags, - key_id, - cost_in_usd, - provider, - model, - status_code, - prompt_token_count, - completion_token_count, - latency_in_ms, - path, - method, - custom_id, - request, - response, - user_id, - action, - policy_id, - route_id, - correlation_id, - metadata - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22 - ) - ` - - tx, err := db.Begin() - if err != nil { - return 0, err - } - defer tx.Rollback() - - stmt, err := tx.Prepare(insertQuery) - if err != nil { - return 0, err - } - defer stmt.Close() - - now := time.Now().UTC() - inserted := 0 - - for dayOffset := 0; dayOffset < cfg.days; dayOffset++ { - day := now.AddDate(0, 0, -dayOffset) - for _, course := range courses { - activityMultiplier := courseActivityMultiplier(course) - for userIndex, userID := range course.users { - if !isUserActiveOnDay(rng, dayOffset, activityMultiplier, userIndex) { - continue - } - - eventCount := sampleEventCount(rng, dayOffset, activityMultiplier, userIndex) - for eventIdx := 0; eventIdx < eventCount; eventIdx++ { - createdAt := sampleTimestamp(rng, day) - costTypeTag, cost := sampleCostProfile(rng, dayOffset, activityMultiplier, userIndex) - promptTokens, completionTokens := sampleTokenCounts(rng, costTypeTag, activityMultiplier) - latency := 200 + rng.Intn(1800) - tags := []string{ - orgTagPrefix + course.orgID, - courseTagPrefix + course.courseID, - userTagPrefix + userID, - costTypeTag, - mockGeneratedTag, - } - - keyID := fmt.Sprintf("mock-key-%s", course.orgID) - customID := fmt.Sprintf("mock-run-%s", day.Format("20060102")) - action := "mock-generate-statistics" - policyID := fmt.Sprintf("mock-policy-%s", course.orgID) - routeID := fmt.Sprintf("mock-route-%s", course.courseID) - correlationID := uuid.NewString() - requestJSON := fmt.Sprintf(`{"mock":true,"tag":"%s","courseId":"%s","userId":"%s"}`, - mockGeneratedTag, - course.courseID, - userID, - ) - responseJSON := `{"ok":true}` - metadataJSON := fmt.Sprintf(`{"generator":"mock-data/generate_statistics_mock.go","dayOffset":%d}`, dayOffset) - - if _, err := stmt.Exec( - uuid.NewString(), - createdAt.Unix(), - pq.Array(tags), - keyID, - cost, - "openai", - sampleModel(costTypeTag), - 200, - promptTokens, - completionTokens, - latency, - "/api/providers/openai/v1/responses", - "POST", - customID, - requestJSON, - responseJSON, - userID, - action, - policyID, - routeID, - correlationID, - metadataJSON, - ); err != nil { - return inserted, err - } - inserted++ - } - } - } - - if cfg.verbose && dayOffset%7 == 0 { - fmt.Printf("generated through day offset %d (%s), inserted=%d\n", dayOffset, day.Format("2006-01-02"), inserted) - } - } - - if err := tx.Commit(); err != nil { - return inserted, err - } - - return inserted, nil -} - -func courseActivityMultiplier(course courseDef) float64 { - switch { - case strings.HasSuffix(course.courseID, "01"): - return 0.8 - case strings.HasSuffix(course.courseID, "02"): - return 1.0 - case strings.HasSuffix(course.courseID, "03"): - return 1.3 - default: - return 1.6 - } -} - -func isUserActiveOnDay(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) bool { - base := 0.25 + multiplier*0.18 - if userIndex < 5 { - base += 0.20 - } - if userIndex > 50 { - base -= 0.08 - } - if dayOffset%7 == 0 || dayOffset%7 == 6 { - base *= 0.75 - } - if dayOffset > 120 { - base *= 0.90 - } - if base > 0.95 { - base = 0.95 - } - if base < 0.05 { - base = 0.05 - } - return rng.Float64() < base -} - -func sampleEventCount(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) int { - count := 1 - if rng.Float64() < 0.45*multiplier { - count++ - } - if rng.Float64() < 0.15*multiplier { - count += 1 + rng.Intn(2) - } - if userIndex < 3 && rng.Float64() < 0.20 { - count += 2 + rng.Intn(4) - } - if dayOffset%30 == 0 && rng.Float64() < 0.30 { - count += 2 - } - if count > 10 { - count = 10 - } - return count -} - -func sampleTimestamp(rng *rand.Rand, day time.Time) time.Time { - base := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, time.UTC) - return base.Add(time.Duration(rng.Intn(24)) * time.Hour). - Add(time.Duration(rng.Intn(60)) * time.Minute). - Add(time.Duration(rng.Intn(60)) * time.Second) -} - -func sampleCostProfile(rng *rand.Rand, dayOffset int, multiplier float64, userIndex int) (string, float64) { - costTypeTag := codioProvidedTag - if rng.Float64() < 0.28 { - costTypeTag = codioSpecialTag - } - - base := 0.08 + rng.Float64()*0.90 - if costTypeTag == codioSpecialTag { - base *= 1.7 - } - base *= multiplier - - if userIndex < 3 { - base *= 2.5 - } - if userIndex >= 3 && userIndex < 10 { - base *= 1.4 - } - - if dayOffset%14 == 0 && rng.Float64() < 0.18 { - base *= 4.0 + rng.Float64()*5.0 - } - if dayOffset%45 == 0 && rng.Float64() < 0.10 { - base *= 8.0 + rng.Float64()*8.0 - } - - return costTypeTag, round2(base) -} - -func sampleTokenCounts(rng *rand.Rand, costTypeTag string, multiplier float64) (int, int) { - prompt := 300 + rng.Intn(2200) - completion := 150 + rng.Intn(1800) - if costTypeTag == codioSpecialTag { - prompt = int(float64(prompt) * (1.2 + multiplier*0.2)) - completion = int(float64(completion) * (1.2 + multiplier*0.2)) - } - return prompt, completion -} - -func sampleModel(costTypeTag string) string { - if costTypeTag == codioSpecialTag { - return "gpt-5.4-mini" - } - return "gpt-5.4-nano" -} - -func round2(v float64) float64 { - return float64(int(v*100+0.5)) / 100 -} From 019121b1ce6eb9b582cc86b2e3f3d77cfcf7d482 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 11 Sep 2026 12:18:01 +0100 Subject: [PATCH 11/14] review --- internal/manager/reporting.go | 10 +++------- internal/storage/redis/statistic-cache.go | 15 ++------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/internal/manager/reporting.go b/internal/manager/reporting.go index ac55d7e..37c9679 100644 --- a/internal/manager/reporting.go +++ b/internal/manager/reporting.go @@ -25,9 +25,8 @@ type keyValidator interface { type StatisticsCache interface { Get(key string) (*event.StatisticsData, error) Set(key string, val *event.StatisticsData, ttl time.Duration) error - SetInProgress(key string) error + TryMarkInProgress(key string) (bool, error) DeleteInProgress(key string) error - IsInProgress(key string) bool } type eventStorage interface { @@ -236,11 +235,8 @@ func (rm *ReportingManager) GetStatistic(r *event.StatisticsRequest) (*event.Sta } func (rm *ReportingManager) backgroundCollectStatisticsData(cacheKey string, level event.StatisticLevel, id *string) { - if rm.sc.IsInProgress(cacheKey) { - return - } - err := rm.sc.SetInProgress(cacheKey) - if err != nil { + claimed, err := rm.sc.TryMarkInProgress(cacheKey) + if err != nil || !claimed { return } defer rm.sc.DeleteInProgress(cacheKey) diff --git a/internal/storage/redis/statistic-cache.go b/internal/storage/redis/statistic-cache.go index 73926a8..e3a7a3b 100644 --- a/internal/storage/redis/statistic-cache.go +++ b/internal/storage/redis/statistic-cache.go @@ -79,22 +79,11 @@ func (c *StatisticCache) Get(key string) (*event.StatisticsData, error) { return &stat, nil } -func (c *StatisticCache) SetInProgress(key string) error { +func (c *StatisticCache) TryMarkInProgress(key string) (bool, error) { k := inProgressKeyPrefix + key ctx, cancel := context.WithTimeout(context.Background(), c.wt) defer cancel() - err := c.client.Set(ctx, k, true, time.Minute*10).Err() - if err != nil { - return err - } - return nil -} - -func (c *StatisticCache) IsInProgress(key string) bool { - k := inProgressKeyPrefix + key - ctx, cancel := context.WithTimeout(context.Background(), c.rt) - defer cancel() - return c.client.Get(ctx, k).Err() == nil + return c.client.SetNX(ctx, k, true, time.Minute*10).Result() } func (c *StatisticCache) DeleteInProgress(key string) error { From 61b041ec270ade64c1e959e33ae786c6308c9d9b Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 11 Sep 2026 14:29:48 +0100 Subject: [PATCH 12/14] remove admin pass --- cmd/bricksllm/main.go | 2 +- internal/server/web/admin/admin.go | 4 ++-- internal/server/web/admin/middleware.go | 8 +------- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cmd/bricksllm/main.go b/cmd/bricksllm/main.go index 009dfce..0dd8ebe 100644 --- a/cmd/bricksllm/main.go +++ b/cmd/bricksllm/main.go @@ -336,7 +336,7 @@ func main() { pm := manager.NewPolicyManager(store, rMemStore) um := manager.NewUserManager(store, store) - as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.AdminPass, cfg.XCodioSignSecret) + as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.XCodioSignSecret) if err != nil { log.Sugar().Fatalf("error creating admin http server: %v", err) } diff --git a/internal/server/web/admin/admin.go b/internal/server/web/admin/admin.go index 048d273..7644a07 100644 --- a/internal/server/web/admin/admin.go +++ b/internal/server/web/admin/admin.go @@ -74,11 +74,11 @@ type AdminServer struct { m KeyManager } -func NewAdminServer(log *zap.Logger, mode string, m KeyManager, krm KeyReportingManager, psm ProviderSettingsManager, cpm CustomProvidersManager, rm RouteManager, pm PoliciesManager, um UserManager, adminPass, xCodioSignSecret string) (*AdminServer, error) { +func NewAdminServer(log *zap.Logger, mode string, m KeyManager, krm KeyReportingManager, psm ProviderSettingsManager, cpm CustomProvidersManager, rm RouteManager, pm PoliciesManager, um UserManager, xCodioSignSecret string) (*AdminServer, error) { router := gin.New() prod := mode == "production" - router.Use(getAdminLoggerMiddleware(log, "admin", prod, adminPass)) + router.Use(getAdminLoggerMiddleware(log, "admin", prod)) router.Use(getAdminSignRequestMiddleware(prod, xCodioSignSecret)) router.GET("/api/health", getGetHealthCheckHandler()) diff --git a/internal/server/web/admin/middleware.go b/internal/server/web/admin/middleware.go index 23498e5..c0c5e45 100644 --- a/internal/server/web/admin/middleware.go +++ b/internal/server/web/admin/middleware.go @@ -15,14 +15,8 @@ import ( "go.uber.org/zap" ) -func getAdminLoggerMiddleware(log *zap.Logger, prefix string, prod bool, adminPass string) gin.HandlerFunc { +func getAdminLoggerMiddleware(log *zap.Logger, prefix string, prod bool) gin.HandlerFunc { return func(c *gin.Context) { - if len(adminPass) != 0 && c.Request.Header.Get("X-API-KEY") != adminPass { - c.Status(200) - c.Abort() - return - } - cid := util.NewUuid() c.Set(util.STRING_CORRELATION_ID, cid) logWithCid := log.With(zap.String(util.STRING_CORRELATION_ID, cid)) From c3367040e7d1ff6dd10e264f854babcf720b5a25 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 11 Sep 2026 16:38:51 +0100 Subject: [PATCH 13/14] tink --- cmd/bricksllm/main.go | 8 +- go.mod | 1 + go.sum | 2 + internal/server/web/admin/admin.go | 4 +- internal/server/web/admin/middleware.go | 30 +++---- .../util/mac-verification/verification.go | 80 +++++++++++++++++++ 6 files changed, 101 insertions(+), 24 deletions(-) create mode 100644 internal/util/mac-verification/verification.go diff --git a/cmd/bricksllm/main.go b/cmd/bricksllm/main.go index 0dd8ebe..f25cf86 100644 --- a/cmd/bricksllm/main.go +++ b/cmd/bricksllm/main.go @@ -322,7 +322,7 @@ func main() { requestsLimitStorage := redisStorage.NewStore(requestsLimitRedisStorage, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) statisticsCache := redisStorage.NewStatisticCache(statisticsRedisCache, cfg.RedisWriteTimeout, cfg.RedisReadTimeout) - encryptor, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience) + encrypt, err := encryptor.NewEncryptor(cfg.DecryptionEndpoint, cfg.EncryptionEndpoint, cfg.EnableEncrytion, cfg.EncryptionTimeout, cfg.Audience) if cfg.EnableEncrytion && err != nil { log.Sugar().Fatalf("error creating encryption client: %v", err) } @@ -330,13 +330,13 @@ func main() { m := manager.NewManager(store, costLimitCache, rateLimitCache, accessCache, keysCache, secondaryKeysCache, requestsLimitStorage) krm := manager.NewReportingManager(costStorage, store, store, v, statisticsCache) - psm := manager.NewProviderSettingsManager(store, psCache, encryptor) + psm := manager.NewProviderSettingsManager(store, psCache, encrypt) cpm := manager.NewCustomProvidersManager(store, cpMemStore) rm := manager.NewRouteManager(store, store, rMemStore, psm) pm := manager.NewPolicyManager(store, rMemStore) um := manager.NewUserManager(store, store) - as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um, cfg.XCodioSignSecret) + as, err := admin.NewAdminServer(log, *modePtr, m, krm, psm, cpm, rm, pm, um) if err != nil { log.Sugar().Fatalf("error creating admin http server: %v", err) } @@ -367,7 +367,7 @@ func main() { rec := recorder.NewRecorder(costStorage, userCostStorage, costLimitCache, userCostLimitCache, ce, store, requestsLimitStorage) rlm := manager.NewRateLimitManager(rateLimitCache, userRateLimitCache) - a := auth.NewAuthenticator(psm, m, rm, store, encryptor) + a := auth.NewAuthenticator(psm, m, rm, store, encrypt) c := cache.NewCache(apiCache) diff --git a/go.mod b/go.mod index 0789b4a..ac0927b 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 github.com/fatih/color v1.18.0 github.com/gin-gonic/gin v1.11.0 + github.com/google/tink/go v1.7.0 github.com/google/uuid v1.6.0 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 diff --git a/go.sum b/go.sum index df2531b..09df7af 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/tink/go v1.7.0 h1:6Eox8zONGebBFcCBqkVmt60LaWZa6xg1cl/DwAh/J1w= +github.com/google/tink/go v1.7.0/go.mod h1:GAUOd+QE3pgj9q8VKIGTCP33c/B7eb4NhxLcgTJZStM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= diff --git a/internal/server/web/admin/admin.go b/internal/server/web/admin/admin.go index 7644a07..fb089ff 100644 --- a/internal/server/web/admin/admin.go +++ b/internal/server/web/admin/admin.go @@ -74,12 +74,12 @@ type AdminServer struct { m KeyManager } -func NewAdminServer(log *zap.Logger, mode string, m KeyManager, krm KeyReportingManager, psm ProviderSettingsManager, cpm CustomProvidersManager, rm RouteManager, pm PoliciesManager, um UserManager, xCodioSignSecret string) (*AdminServer, error) { +func NewAdminServer(log *zap.Logger, mode string, m KeyManager, krm KeyReportingManager, psm ProviderSettingsManager, cpm CustomProvidersManager, rm RouteManager, pm PoliciesManager, um UserManager) (*AdminServer, error) { router := gin.New() prod := mode == "production" router.Use(getAdminLoggerMiddleware(log, "admin", prod)) - router.Use(getAdminSignRequestMiddleware(prod, xCodioSignSecret)) + router.Use(getAdminSignRequestMiddleware(prod)) router.GET("/api/health", getGetHealthCheckHandler()) diff --git a/internal/server/web/admin/middleware.go b/internal/server/web/admin/middleware.go index c0c5e45..c050363 100644 --- a/internal/server/web/admin/middleware.go +++ b/internal/server/web/admin/middleware.go @@ -2,15 +2,13 @@ package admin import ( "bytes" - "crypto/hmac" - "crypto/sha1" - "encoding/base64" "fmt" "io" "net/http" "time" "github.com/bricks-cloud/bricksllm/internal/util" + macverification "github.com/bricks-cloud/bricksllm/internal/util/mac-verification" "github.com/gin-gonic/gin" "go.uber.org/zap" ) @@ -34,13 +32,13 @@ func getAdminLoggerMiddleware(log *zap.Logger, prefix string, prod bool) gin.Han zap.Int("code", c.Writer.Status()), zap.String("method", c.Request.Method), zap.String("path", c.FullPath()), - zap.Int64("lantecyInMs", latency), + zap.Int64("latencyInMs", latency), ) } } } -func getAdminSignRequestMiddleware(prod bool, xCodioSignSecret string) gin.HandlerFunc { +func getAdminSignRequestMiddleware(prod bool) gin.HandlerFunc { return func(c *gin.Context) { log := util.GetLogFromCtx(c) @@ -48,9 +46,12 @@ func getAdminSignRequestMiddleware(prod bool, xCodioSignSecret string) gin.Handl c.Next() return } - sign := c.Request.Header.Get("X-Codio-Sign") - timestamp := c.Request.Header.Get("X-Codio-Sign-Timestamp") - if len(sign) == 0 || len(timestamp) == 0 || len(xCodioSignSecret) == 0 { + + timestamp := c.GetHeader("X-Codio-Sign-Timestamp") + token := c.GetHeader("X-Codio-Sign") + provider := c.GetHeader("X-Codio-Provider") + + if len(token) == 0 || len(timestamp) == 0 || len(provider) == 0 { c.Status(403) c.Abort() return @@ -68,9 +69,10 @@ func getAdminSignRequestMiddleware(prod bool, xCodioSignSecret string) gin.Handl }) return } - data := fmt.Sprintf("%s%s", timestamp, body) + data := fmt.Sprintf("%s%s%s", timestamp, body, provider) c.Request.Body = io.NopCloser(bytes.NewReader(body)) - if !validSign([]byte(data), []byte(xCodioSignSecret), sign) { + + if !macverification.VerifySign(provider, []byte(data), token) { c.Status(403) c.Abort() return @@ -78,11 +80,3 @@ func getAdminSignRequestMiddleware(prod bool, xCodioSignSecret string) gin.Handl c.Next() } } - -func validSign(message, key []byte, messageSign string) bool { - mac := hmac.New(sha1.New, key) - mac.Write(message) - expectedMAC := mac.Sum(nil) - expectedSign := base64.StdEncoding.EncodeToString(expectedMAC) - return messageSign == expectedSign -} diff --git a/internal/util/mac-verification/verification.go b/internal/util/mac-verification/verification.go new file mode 100644 index 0000000..1989d95 --- /dev/null +++ b/internal/util/mac-verification/verification.go @@ -0,0 +1,80 @@ +package mac_verification + +import ( + "encoding/base64" + "fmt" + "log" + "os" + "path/filepath" + + "github.com/google/tink/go/insecurecleartextkeyset" + "github.com/google/tink/go/keyset" + "github.com/google/tink/go/mac" + "github.com/google/tink/go/tink" +) + +const keySetDir = "./tink" + +var macCache map[string]tink.MAC = make(map[string]tink.MAC) + +func init() { + err := loadKeySets() + if err != nil { + log.Fatal(err) + } +} + +func loadKeySets() error { + files, err := os.ReadDir(keySetDir) + if err != nil { + return fmt.Errorf("error reading directory %s: %v", keySetDir, err) + } + for _, file := range files { + if file.IsDir() { + continue + } + filePath := filepath.Join(keySetDir, file.Name()) + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("error opening keyset file %s: %v", filePath, err) + } + defer f.Close() + + jsonReader := keyset.NewJSONReader(f) + + kh, err := insecurecleartextkeyset.Read(jsonReader) + if err != nil { + return fmt.Errorf("error reading keyset from file %s: %v", filePath, err) + } + + macPrimitive, err := mac.New(kh) + if err != nil { + return fmt.Errorf("error creating MAC primitive from keyset %s: %v", filePath, err) + } + + providerName := file.Name() + if len(providerName) > 5 && providerName[len(providerName)-5:] == ".json" { + providerName = providerName[:len(providerName)-5] + } + macCache[providerName] = macPrimitive + } + return nil +} + +func VerifySign(provider string, data []byte, token string) bool { + decodedSignature, err := base64.StdEncoding.DecodeString(token) + if err != nil { + fmt.Printf("Error decoding token from Base64: %v\n", err) + return false + } + macPrimitive, ok := macCache[provider] + if !ok { + fmt.Printf("MAC primitive for provider %s not found\n", provider) + return false + } + err = macPrimitive.VerifyMAC(decodedSignature, data) + if err != nil { + return false + } + return true +} From fc387fb12d32d79a1bf83ecaa69e6ca4fbaf3637 Mon Sep 17 00:00:00 2001 From: Sergei Bronnikov Date: Fri, 11 Sep 2026 18:11:50 +0100 Subject: [PATCH 14/14] keySetDir --- internal/util/mac-verification/verification.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/util/mac-verification/verification.go b/internal/util/mac-verification/verification.go index 1989d95..f318177 100644 --- a/internal/util/mac-verification/verification.go +++ b/internal/util/mac-verification/verification.go @@ -13,7 +13,7 @@ import ( "github.com/google/tink/go/tink" ) -const keySetDir = "./tink" +const keySetSubDir = "./tink" var macCache map[string]tink.MAC = make(map[string]tink.MAC) @@ -25,6 +25,12 @@ func init() { } func loadKeySets() error { + execDir, err := os.Executable() + if err != nil { + return fmt.Errorf("error getting executable path: %v", err) + } + execDir = filepath.Dir(execDir) + keySetDir := filepath.Join(execDir, keySetSubDir) files, err := os.ReadDir(keySetDir) if err != nil { return fmt.Errorf("error reading directory %s: %v", keySetDir, err)