Fix how we query scores to be more performant

cmd/console/cmd.go

@@ -51,6 +51,7 @@ func init() {
 	RootCmd.AddCommand(commands.ClanRankMapCmd)
 	RootCmd.AddCommand(commands.DenyOnHoldCmd)
 	RootCmd.AddCommand(commands.DatabaseScoresBatchDeleteFailed)
+	RootCmd.AddCommand(commands.DatabaseScoresModsBackfill)
 	RootCmd.AddCommand(commands.DeleteScoreCmd)
 	RootCmd.AddCommand(commands.FixStatsCmd)
 	RootCmd.AddCommand(commands.UpdateStripePriceId)

cmd/console/commands/database_scores_mods_backfill.go

@@ -0,0 +1,110 @@
+package commands
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/Quaver/api2/db"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gorm.io/gorm"
+)
+
+var scoreModsBackfillBatchSize int
+var scoreModsBackfillSleepMs int
+var scoreModsBackfillMaxBatches int64
+
+var DatabaseScoresModsBackfill = &cobra.Command{
+	Use:   "database:scores:mods:backfill",
+	Short: "Backfills score modifier lookup columns in batches",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if scoreModsBackfillBatchSize <= 0 {
+			return fmt.Errorf("batch-size must be greater than 0")
+		}
+
+		if scoreModsBackfillSleepMs < 0 {
+			return fmt.Errorf("sleep-ms cannot be negative")
+		}
+
+		if scoreModsBackfillMaxBatches < 0 {
+			return fmt.Errorf("max-batches cannot be negative")
+		}
+
+		var completedBatches int64
+		var totalUpdated int64
+		lastScoreID := 0
+
+		logrus.Infof(
+			"Backfilling score modifier columns for all maps with batches of %d",
+			scoreModsBackfillBatchSize,
+		)
+
+		for {
+			if scoreModsBackfillMaxBatches > 0 && completedBatches >= scoreModsBackfillMaxBatches {
+				logrus.Infof(
+					"Reached max-batches=%d after updating %d rows. Run the command again to resume.",
+					scoreModsBackfillMaxBatches,
+					totalUpdated,
+				)
+				return nil
+			}
+
+			var scoreIDs []int
+
+			result := db.SQL.WithContext(cmd.Context()).
+				Model(&db.Score{}).
+				Where("mods_migrated = 0 AND id > ?", lastScoreID).
+				Order("id ASC").
+				Limit(scoreModsBackfillBatchSize).
+				Pluck("id", &scoreIDs)
+
+			if result.Error != nil {
+				return fmt.Errorf("retrieve unmigrated scores after score id %d: %w", lastScoreID, result.Error)
+			}
+
+			if len(scoreIDs) == 0 {
+				break
+			}
+
+			result = db.SQL.WithContext(cmd.Context()).
+				Model(&db.Score{}).
+				Where("id IN ? AND mods_migrated = 0", scoreIDs).
+				UpdateColumn("mods", gorm.Expr("mods"))
+
+			if result.Error != nil {
+				return fmt.Errorf("backfill score modifier columns through score id %d: %w", scoreIDs[len(scoreIDs)-1], result.Error)
+			}
+
+			lastScoreID = scoreIDs[len(scoreIDs)-1]
+			completedBatches++
+			totalUpdated += result.RowsAffected
+
+			logrus.Infof(
+				"Backfilled score modifier columns through score id %d (%d rows, %d total, %d batches complete)",
+				lastScoreID,
+				result.RowsAffected,
+				totalUpdated,
+				completedBatches,
+			)
+
+			if scoreModsBackfillSleepMs > 0 {
+				time.Sleep(time.Duration(scoreModsBackfillSleepMs) * time.Millisecond)
+			}
+		}
+
+		logrus.Infof(
+			"Score modifier column backfill complete. Processed %d batches and updated %d rows.",
+			completedBatches,
+			totalUpdated,
+		)
+		logrus.Info("Set score_mod_columns_ready=true and restart all API instances to enable indexed score modifier lookups.")
+
+		return nil
+	},
+}
+
+func init() {
+	DatabaseScoresModsBackfill.Flags().IntVar(&scoreModsBackfillBatchSize, "batch-size", 5000, "Maximum number of scores to backfill per batch")
+	DatabaseScoresModsBackfill.Flags().IntVar(&scoreModsBackfillSleepMs, "sleep-ms", 0, "Milliseconds to sleep between batches")
+	DatabaseScoresModsBackfill.Flags().Int64Var(&scoreModsBackfillMaxBatches, "max-batches", 0, "Maximum number of batches to process before exiting")
+}

cmd/console/commands/database_scores_mods_backfill_test.go

@@ -0,0 +1,54 @@
+package commands
+
+import "testing"
+
+func TestDatabaseScoresModsBackfillValidation(t *testing.T) {
+	originalBatchSize := scoreModsBackfillBatchSize
+	originalSleepMs := scoreModsBackfillSleepMs
+	originalMaxBatches := scoreModsBackfillMaxBatches
+
+	t.Cleanup(func() {
+		scoreModsBackfillBatchSize = originalBatchSize
+		scoreModsBackfillSleepMs = originalSleepMs
+		scoreModsBackfillMaxBatches = originalMaxBatches
+	})
+
+	tests := []struct {
+		name       string
+		batchSize  int
+		sleepMs    int
+		maxBatches int64
+		wantError  string
+	}{
+		{
+			name:      "zero batch size",
+			batchSize: 0,
+			wantError: "batch-size must be greater than 0",
+		},
+		{
+			name:      "negative sleep",
+			batchSize: 1,
+			sleepMs:   -1,
+			wantError: "sleep-ms cannot be negative",
+		},
+		{
+			name:       "negative max batches",
+			batchSize:  1,
+			maxBatches: -1,
+			wantError:  "max-batches cannot be negative",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			scoreModsBackfillBatchSize = tt.batchSize
+			scoreModsBackfillSleepMs = tt.sleepMs
+			scoreModsBackfillMaxBatches = tt.maxBatches
+			err := DatabaseScoresModsBackfill.RunE(DatabaseScoresModsBackfill, nil)
+
+			if err == nil || err.Error() != tt.wantError {
+				t.Fatalf("RunE() error = %v, want %q", err, tt.wantError)
+			}
+		})
+	}
+}

cmd/database/migrations/29_score_mod_columns.down.sql

@@ -0,0 +1,12 @@
+DROP TRIGGER IF EXISTS scores_mod_columns_bu;
+DROP TRIGGER IF EXISTS scores_mod_columns_bi;
+
+ALTER TABLE scores
+    DROP COLUMN mods_migrated,
+    DROP COLUMN no_miss,
+    DROP COLUMN full_ln,
+    DROP COLUMN inverse,
+    DROP COLUMN no_long_notes,
+    DROP COLUMN no_slider_velocities,
+    DROP COLUMN mirror,
+    DROP COLUMN speed_rate;

cmd/database/migrations/29_score_mod_columns.up.sql

@@ -0,0 +1,97 @@
+ALTER TABLE scores
+    ADD COLUMN speed_rate SMALLINT UNSIGNED NULL AFTER mods,
+    ADD COLUMN mirror TINYINT(1) NULL AFTER speed_rate,
+    ADD COLUMN no_slider_velocities TINYINT(1) NULL AFTER mirror,
+    ADD COLUMN no_long_notes TINYINT(1) NULL AFTER no_slider_velocities,
+    ADD COLUMN inverse TINYINT(1) NULL AFTER no_long_notes,
+    ADD COLUMN full_ln TINYINT(1) NULL AFTER inverse,
+    ADD COLUMN no_miss TINYINT(1) NULL AFTER full_ln,
+    ADD COLUMN mods_migrated TINYINT(1) NOT NULL DEFAULT 0 AFTER no_miss;
+
+CREATE TRIGGER scores_mod_columns_bi
+    BEFORE INSERT ON scores
+    FOR EACH ROW
+    SET NEW.speed_rate = CASE
+            WHEN (NEW.mods & 2) != 0 THEN 50
+            WHEN (NEW.mods & 16777216) != 0 THEN 55
+            WHEN (NEW.mods & 4) != 0 THEN 60
+            WHEN (NEW.mods & 33554432) != 0 THEN 65
+            WHEN (NEW.mods & 8) != 0 THEN 70
+            WHEN (NEW.mods & 67108864) != 0 THEN 75
+            WHEN (NEW.mods & 16) != 0 THEN 80
+            WHEN (NEW.mods & 134217728) != 0 THEN 85
+            WHEN (NEW.mods & 32) != 0 THEN 90
+            WHEN (NEW.mods & 268435456) != 0 THEN 95
+            WHEN (NEW.mods & 8589934592) != 0 THEN 105
+            WHEN (NEW.mods & 64) != 0 THEN 110
+            WHEN (NEW.mods & 17179869184) != 0 THEN 115
+            WHEN (NEW.mods & 128) != 0 THEN 120
+            WHEN (NEW.mods & 34359738368) != 0 THEN 125
+            WHEN (NEW.mods & 256) != 0 THEN 130
+            WHEN (NEW.mods & 68719476736) != 0 THEN 135
+            WHEN (NEW.mods & 512) != 0 THEN 140
+            WHEN (NEW.mods & 137438953472) != 0 THEN 145
+            WHEN (NEW.mods & 1024) != 0 THEN 150
+            WHEN (NEW.mods & 274877906944) != 0 THEN 155
+            WHEN (NEW.mods & 2048) != 0 THEN 160
+            WHEN (NEW.mods & 549755813888) != 0 THEN 165
+            WHEN (NEW.mods & 4096) != 0 THEN 170
+            WHEN (NEW.mods & 1099511627776) != 0 THEN 175
+            WHEN (NEW.mods & 8192) != 0 THEN 180
+            WHEN (NEW.mods & 2199023255552) != 0 THEN 185
+            WHEN (NEW.mods & 16384) != 0 THEN 190
+            WHEN (NEW.mods & 4398046511104) != 0 THEN 195
+            WHEN (NEW.mods & 32768) != 0 THEN 200
+            ELSE 100
+        END,
+        NEW.mirror = IF((NEW.mods & 2147483648) != 0, 1, 0),
+        NEW.no_slider_velocities = IF((NEW.mods & 1) != 0, 1, 0),
+        NEW.no_long_notes = IF((NEW.mods & 4194304) != 0, 1, 0),
+        NEW.inverse = IF((NEW.mods & 536870912) != 0, 1, 0),
+        NEW.full_ln = IF((NEW.mods & 1073741824) != 0, 1, 0),
+        NEW.no_miss = IF((NEW.mods & 17592186044416) != 0, 1, 0),
+        NEW.mods_migrated = 1;
+
+CREATE TRIGGER scores_mod_columns_bu
+    BEFORE UPDATE ON scores
+    FOR EACH ROW
+    SET NEW.speed_rate = CASE
+            WHEN (NEW.mods & 2) != 0 THEN 50
+            WHEN (NEW.mods & 16777216) != 0 THEN 55
+            WHEN (NEW.mods & 4) != 0 THEN 60
+            WHEN (NEW.mods & 33554432) != 0 THEN 65
+            WHEN (NEW.mods & 8) != 0 THEN 70
+            WHEN (NEW.mods & 67108864) != 0 THEN 75
+            WHEN (NEW.mods & 16) != 0 THEN 80
+            WHEN (NEW.mods & 134217728) != 0 THEN 85
+            WHEN (NEW.mods & 32) != 0 THEN 90
+            WHEN (NEW.mods & 268435456) != 0 THEN 95
+            WHEN (NEW.mods & 8589934592) != 0 THEN 105
+            WHEN (NEW.mods & 64) != 0 THEN 110
+            WHEN (NEW.mods & 17179869184) != 0 THEN 115
+            WHEN (NEW.mods & 128) != 0 THEN 120
+            WHEN (NEW.mods & 34359738368) != 0 THEN 125
+            WHEN (NEW.mods & 256) != 0 THEN 130
+            WHEN (NEW.mods & 68719476736) != 0 THEN 135
+            WHEN (NEW.mods & 512) != 0 THEN 140
+            WHEN (NEW.mods & 137438953472) != 0 THEN 145
+            WHEN (NEW.mods & 1024) != 0 THEN 150
+            WHEN (NEW.mods & 274877906944) != 0 THEN 155
+            WHEN (NEW.mods & 2048) != 0 THEN 160
+            WHEN (NEW.mods & 549755813888) != 0 THEN 165
+            WHEN (NEW.mods & 4096) != 0 THEN 170
+            WHEN (NEW.mods & 1099511627776) != 0 THEN 175
+            WHEN (NEW.mods & 8192) != 0 THEN 180
+            WHEN (NEW.mods & 2199023255552) != 0 THEN 185
+            WHEN (NEW.mods & 16384) != 0 THEN 190
+            WHEN (NEW.mods & 4398046511104) != 0 THEN 195
+            WHEN (NEW.mods & 32768) != 0 THEN 200
+            ELSE 100
+        END,
+        NEW.mirror = IF((NEW.mods & 2147483648) != 0, 1, 0),
+        NEW.no_slider_velocities = IF((NEW.mods & 1) != 0, 1, 0),
+        NEW.no_long_notes = IF((NEW.mods & 4194304) != 0, 1, 0),
+        NEW.inverse = IF((NEW.mods & 536870912) != 0, 1, 0),
+        NEW.full_ln = IF((NEW.mods & 1073741824) != 0, 1, 0),
+        NEW.no_miss = IF((NEW.mods & 17592186044416) != 0, 1, 0),
+        NEW.mods_migrated = 1;

cmd/database/migrations/30_score_mod_column_indexes.down.sql

@@ -0,0 +1,3 @@
+DROP INDEX scores_mod_no_miss_idx ON scores;
+DROP INDEX scores_mod_mirror_idx ON scores;
+DROP INDEX scores_mod_speed_rate_idx ON scores;

cmd/database/migrations/30_score_mod_column_indexes.up.sql

@@ -0,0 +1,8 @@
+CREATE INDEX scores_mod_speed_rate_idx
+    ON scores (map_md5, failed, speed_rate, user_id, performance_rating DESC, timestamp DESC);
+
+CREATE INDEX scores_mod_mirror_idx
+    ON scores (map_md5, failed, mirror, user_id, performance_rating DESC, timestamp DESC);
+
+CREATE INDEX scores_mod_no_miss_idx
+    ON scores (map_md5, failed, no_miss, user_id, performance_rating DESC, timestamp DESC);

cmd/database/migrations/32_scores_global_scoreboard_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX scores_global_scoreboard_idx ON scores;

cmd/database/migrations/32_scores_global_scoreboard_index.up.sql

@@ -0,0 +1,2 @@
+CREATE INDEX scores_global_scoreboard_idx
+    ON scores (map_md5, personal_best, performance_rating DESC, user_id);

cmd/database/migrations/33_scores_index_cleanup.down.sql

@@ -0,0 +1,8 @@
+CREATE UNIQUE INDEX `UNIQUE`
+    ON scores (id);
+
+CREATE INDEX personal_best
+    ON scores (personal_best);
+
+CREATE INDEX mods
+    ON scores (mods);

cmd/database/migrations/33_scores_index_cleanup.up.sql

@@ -0,0 +1,7 @@
+DROP INDEX IF EXISTS `UNIQUE` ON scores;
+DROP INDEX IF EXISTS personal_best ON scores;
+DROP INDEX IF EXISTS mods ON scores;
+DROP INDEX IF EXISTS scores_mod_nsv_idx ON scores;
+DROP INDEX IF EXISTS scores_mod_nln_idx ON scores;
+DROP INDEX IF EXISTS scores_mod_inverse_idx ON scores;
+DROP INDEX IF EXISTS scores_mod_full_ln_idx ON scores;

config.example.json

@@ -1,5 +1,6 @@
 {
   "is_production": false,
+  "score_mod_columns_ready": false,
   "api_url": "https://api.quavergame.com",
   "website_url": "http://localhost:8081",
   "jwt_secret": "",
@@ -140,4 +141,4 @@
       "schedule": "0 * * * *"
     }
   }
-}
+}

config/config.go

@@ -9,7 +9,8 @@ import (
 )
 
 type Config struct {
-	IsProduction bool `json:"is_production"`
+	IsProduction         bool `json:"is_production"`
+	ScoreModColumnsReady bool `json:"score_mod_columns_ready"`
 
 	APIUrl string `json:"api_url"`