From b25ea01ad52f0bbbb146c7636735eca358d3dc84 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 27 Sep 2026 22:37:46 -0700 Subject: [PATCH] perf(search): adopt crawlkit 0.16.6 vector speedups --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- internal/store/query.go | 9 +- internal/store/semantic_benchmark_test.go | 106 ++++++++++++++++++++++ internal/store/store_test.go | 37 ++++++++ 6 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 internal/store/semantic_benchmark_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5331a608..ff780697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Speed up exact semantic search 3.7× (10,000 × 1,536-dimensional embeddings: 506 ms → 137 ms, allocations 15.5M → 140k) by updating Crawlkit to v0.16.6 and skipping a redundant norm pass per candidate. - Distinguish cloud login and publisher authorization failures with bounded diagnostics that preserve private response details. - Select the actual latest metrics observation at nanosecond precision, including imported timestamps with different offsets, without rewriting retained history. - Stop cloud adoption when inventory pagination repeats a cursor, and release unread HTTP error responses before reporting publication failures. diff --git a/go.mod b/go.mod index f598817f..1bbbaa2a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/alecthomas/kong v1.16.1 github.com/bwmarrin/discordgo v0.29.0 github.com/gorilla/websocket v1.5.3 - github.com/openclaw/crawlkit v0.16.4 + github.com/openclaw/crawlkit v0.16.6 github.com/stretchr/testify v1.12.1 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 3c52312e..662aeb21 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/openclaw/crawlkit v0.16.4 h1:9pE9nQDOX4nHVKvkfOFeZTBiK+9dlPrJ1RhSjZKdfpI= -github.com/openclaw/crawlkit v0.16.4/go.mod h1:EfLf+VfxQZsMbrri0Sq0S4mCyhuZOHBizNISGuZXjSQ= +github.com/openclaw/crawlkit v0.16.6 h1:Jq2hvcy0ugIkxd+GfNnMPryvYbIsONyg05rVMNbTLfU= +github.com/openclaw/crawlkit v0.16.6/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= diff --git a/internal/store/query.go b/internal/store/query.go index 5cba295d..74a16013 100644 --- a/internal/store/query.go +++ b/internal/store/query.go @@ -497,18 +497,21 @@ func (s *Store) SearchMessagesSemantic(ctx context.Context, opts SemanticSearchO if len(storedVector) != dimensions { return nil, fmt.Errorf("stored embedding vector length mismatch for message %s: got %d want %d", messageID, len(storedVector), dimensions) } - if vector.Norm(storedVector) == 0 { - return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID) - } candidate := semanticSearchCandidate{messageID: messageID, createdAt: parseTime(created)} switch opts.VectorBackend { case vector.BackendExact: score, err := vector.CosineSimilarity(opts.QueryVector, queryNorm, storedVector) if err != nil { + if vector.Norm(storedVector) == 0 { + return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID) + } return nil, fmt.Errorf("score embedding for message %s: %w", messageID, err) } ranked = insertSemanticRanked(ranked, semanticRankedResult{candidate: candidate, score: score}, opts.Limit) case vector.BackendTurboVec: + if vector.Norm(storedVector) == 0 { + return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID) + } turboBatch = append(turboBatch, vector.SearchCandidate[semanticSearchCandidate]{ Item: candidate, Vector: storedVector, diff --git a/internal/store/semantic_benchmark_test.go b/internal/store/semantic_benchmark_test.go new file mode 100644 index 00000000..94dd72e4 --- /dev/null +++ b/internal/store/semantic_benchmark_test.go @@ -0,0 +1,106 @@ +package store + +import ( + "context" + "math" + "math/rand/v2" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// BenchmarkSearchMessagesSemanticExact includes SQLite scanning, vector decoding, +// scoring, ranking and result hydration; fixture creation is outside the timer. +func BenchmarkSearchMessagesSemanticExact(b *testing.B) { + const ( + candidates = 10_000 + dimensions = 1536 + limit = 20 + ) + ctx := context.Background() + s, err := Open(ctx, filepath.Join(b.TempDir(), "discrawl.db")) + require.NoError(b, err) + b.Cleanup(func() { require.NoError(b, s.Close()) }) + + require.NoError(b, s.UpsertGuild(ctx, GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`})) + require.NoError(b, s.UpsertChannel(ctx, ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`})) + base := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC) + messages := make([]MessageMutation, candidates) + for i := range messages { + content := "Archived Discord message " + strconv.Itoa(i) + messages[i].Record = MessageRecord{ + ID: "m" + strconv.Itoa(i), + GuildID: "g1", + ChannelID: "c1", + AuthorID: "u1", + CreatedAt: base.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano), + Content: content, + NormalizedContent: content, + RawJSON: `{"author":{"username":"benchmark"}}`, + } + } + require.NoError(b, s.UpsertMessages(ctx, messages)) + + tx, err := s.DB().BeginTx(ctx, nil) + require.NoError(b, err) + defer func() { _ = tx.Rollback() }() + stmt, err := tx.PrepareContext(ctx, ` + insert into message_embeddings( + message_id, provider, model, input_version, dimensions, embedding_blob, embedded_at + ) values(?, ?, ?, ?, ?, ?, ?) + `) + require.NoError(b, err) + defer func() { _ = stmt.Close() }() + + rng := rand.New(rand.NewPCG(1, 2)) + var query []float32 + for _, message := range messages { + values := make([]float32, dimensions) + var squaredNorm float64 + for j := range values { + values[j] = float32(rng.NormFloat64()) + squaredNorm += float64(values[j]) * float64(values[j]) + } + norm := math.Sqrt(squaredNorm) + for j := range values { + values[j] = float32(float64(values[j]) / norm) + } + if query == nil { + query = values + } + blob, err := EncodeEmbeddingVector(values) + require.NoError(b, err) + _, err = stmt.ExecContext(ctx, message.Record.ID, "benchmark", "dense-1536", EmbeddingInputVersion, dimensions, blob, base.Format(timeLayout)) + require.NoError(b, err) + } + require.NoError(b, tx.Commit()) + + opts := SemanticSearchOptions{ + QueryVector: query, + Provider: "benchmark", + Model: "dense-1536", + InputVersion: EmbeddingInputVersion, + Dimensions: dimensions, + VectorBackend: "exact", + GuildIDs: []string{"g1"}, + Limit: limit, + } + results, err := s.SearchMessagesSemantic(ctx, opts) + require.NoError(b, err) + require.Len(b, results, limit) + require.Equal(b, "m0", results[0].MessageID) + + b.ReportAllocs() + for b.Loop() { + results, err = s.SearchMessagesSemantic(ctx, opts) + if err != nil { + b.Fatal(err) + } + if len(results) != limit || results[0].MessageID != "m0" { + b.Fatal("unexpected semantic search results") + } + } +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index e1863af1..8241b14d 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1042,6 +1042,43 @@ func TestSearchMessagesSemanticErrors(t *testing.T) { require.ErrorContains(t, err, "turbovec dimensions must be a positive multiple of 8") } +func TestSearchMessagesSemanticZeroStoredVector(t *testing.T) { + t.Parallel() + + for _, backend := range []string{"exact", "turbovec"} { + t.Run(backend, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + defer func() { _ = s.Close() }() + + require.NoError(t, s.UpsertMessage(ctx, MessageRecord{ + ID: "zero-vector", + GuildID: "g1", + ChannelID: "c1", + CreatedAt: "2026-09-27T12:00:00Z", + Content: "corrupt embedding", + NormalizedContent: "corrupt embedding", + RawJSON: `{}`, + })) + require.NoError(t, insertTestEmbedding(ctx, s, "zero-vector", "ollama", "nomic-embed-text", make([]float32, 8))) + + _, err = s.SearchMessagesSemantic(ctx, SemanticSearchOptions{ + QueryVector: []float32{1, 0, 0, 0, 0, 0, 0, 0}, + Provider: "ollama", + Model: "nomic-embed-text", + InputVersion: EmbeddingInputVersion, + Dimensions: 8, + VectorBackend: backend, + Limit: 20, + }) + require.EqualError(t, err, "score embedding for message zero-vector: stored embedding vector is zero") + }) + } +} + func TestSearchMessagesHybridFusesAndDeduplicates(t *testing.T) { t.Parallel()