improve(search): adopt Crawlkit v0.16.6 vector speedups

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Speed up exact semantic search 3.7× (10,000 × 1,536-dimensional embeddings: 506 ms → 137 ms, allocations 15.5M → 140k) by updating Crawlkit to v0.16.6 and skipping a redundant norm pass per candidate.
 - Distinguish cloud login and publisher authorization failures with bounded diagnostics that preserve private response details.
 - Select the actual latest metrics observation at nanosecond precision, including imported timestamps with different offsets, without rewriting retained history.
 - Stop cloud adoption when inventory pagination repeats a cursor, and release unread HTTP error responses before reporting publication failures.

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/alecthomas/kong v1.16.1
 	github.com/bwmarrin/discordgo v0.29.0
 	github.com/gorilla/websocket v1.5.3
-	github.com/openclaw/crawlkit v0.16.4
+	github.com/openclaw/crawlkit v0.16.6
 	github.com/stretchr/testify v1.12.1
 	github.com/zalando/go-keyring v0.2.8
 	golang.org/x/sys v0.48.0

go.sum

@@ -61,8 +61,8 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
 github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
-github.com/openclaw/crawlkit v0.16.4 h1:9pE9nQDOX4nHVKvkfOFeZTBiK+9dlPrJ1RhSjZKdfpI=
-github.com/openclaw/crawlkit v0.16.4/go.mod h1:EfLf+VfxQZsMbrri0Sq0S4mCyhuZOHBizNISGuZXjSQ=
+github.com/openclaw/crawlkit v0.16.6 h1:Jq2hvcy0ugIkxd+GfNnMPryvYbIsONyg05rVMNbTLfU=
+github.com/openclaw/crawlkit v0.16.6/go.mod h1:XTGhBPNiMqKzPuKNwAH9ufbUU0Vel0pEQAZFqS/b7GQ=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
 github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=

internal/store/query.go

@@ -497,18 +497,21 @@ func (s *Store) SearchMessagesSemantic(ctx context.Context, opts SemanticSearchO
 		if len(storedVector) != dimensions {
 			return nil, fmt.Errorf("stored embedding vector length mismatch for message %s: got %d want %d", messageID, len(storedVector), dimensions)
 		}
-		if vector.Norm(storedVector) == 0 {
-			return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID)
-		}
 		candidate := semanticSearchCandidate{messageID: messageID, createdAt: parseTime(created)}
 		switch opts.VectorBackend {
 		case vector.BackendExact:
 			score, err := vector.CosineSimilarity(opts.QueryVector, queryNorm, storedVector)
 			if err != nil {
+				if vector.Norm(storedVector) == 0 {
+					return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID)
+				}
 				return nil, fmt.Errorf("score embedding for message %s: %w", messageID, err)
 			}
 			ranked = insertSemanticRanked(ranked, semanticRankedResult{candidate: candidate, score: score}, opts.Limit)
 		case vector.BackendTurboVec:
+			if vector.Norm(storedVector) == 0 {
+				return nil, fmt.Errorf("score embedding for message %s: stored embedding vector is zero", messageID)
+			}
 			turboBatch = append(turboBatch, vector.SearchCandidate[semanticSearchCandidate]{
 				Item:   candidate,
 				Vector: storedVector,

internal/store/semantic_benchmark_test.go

@@ -0,0 +1,106 @@
+package store
+
+import (
+	"context"
+	"math"
+	"math/rand/v2"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+// BenchmarkSearchMessagesSemanticExact includes SQLite scanning, vector decoding,
+// scoring, ranking and result hydration; fixture creation is outside the timer.
+func BenchmarkSearchMessagesSemanticExact(b *testing.B) {
+	const (
+		candidates = 10_000
+		dimensions = 1536
+		limit      = 20
+	)
+	ctx := context.Background()
+	s, err := Open(ctx, filepath.Join(b.TempDir(), "discrawl.db"))
+	require.NoError(b, err)
+	b.Cleanup(func() { require.NoError(b, s.Close()) })
+
+	require.NoError(b, s.UpsertGuild(ctx, GuildRecord{ID: "g1", Name: "Guild", RawJSON: `{}`}))
+	require.NoError(b, s.UpsertChannel(ctx, ChannelRecord{ID: "c1", GuildID: "g1", Kind: "text", Name: "general", RawJSON: `{}`}))
+	base := time.Date(2026, 9, 27, 12, 0, 0, 0, time.UTC)
+	messages := make([]MessageMutation, candidates)
+	for i := range messages {
+		content := "Archived Discord message " + strconv.Itoa(i)
+		messages[i].Record = MessageRecord{
+			ID:                "m" + strconv.Itoa(i),
+			GuildID:           "g1",
+			ChannelID:         "c1",
+			AuthorID:          "u1",
+			CreatedAt:         base.Add(time.Duration(i) * time.Second).Format(time.RFC3339Nano),
+			Content:           content,
+			NormalizedContent: content,
+			RawJSON:           `{"author":{"username":"benchmark"}}`,
+		}
+	}
+	require.NoError(b, s.UpsertMessages(ctx, messages))
+
+	tx, err := s.DB().BeginTx(ctx, nil)
+	require.NoError(b, err)
+	defer func() { _ = tx.Rollback() }()
+	stmt, err := tx.PrepareContext(ctx, `
+		insert into message_embeddings(
+			message_id, provider, model, input_version, dimensions, embedding_blob, embedded_at
+		) values(?, ?, ?, ?, ?, ?, ?)
+	`)
+	require.NoError(b, err)
+	defer func() { _ = stmt.Close() }()
+
+	rng := rand.New(rand.NewPCG(1, 2))
+	var query []float32
+	for _, message := range messages {
+		values := make([]float32, dimensions)
+		var squaredNorm float64
+		for j := range values {
+			values[j] = float32(rng.NormFloat64())
+			squaredNorm += float64(values[j]) * float64(values[j])
+		}
+		norm := math.Sqrt(squaredNorm)
+		for j := range values {
+			values[j] = float32(float64(values[j]) / norm)
+		}
+		if query == nil {
+			query = values
+		}
+		blob, err := EncodeEmbeddingVector(values)
+		require.NoError(b, err)
+		_, err = stmt.ExecContext(ctx, message.Record.ID, "benchmark", "dense-1536", EmbeddingInputVersion, dimensions, blob, base.Format(timeLayout))
+		require.NoError(b, err)
+	}
+	require.NoError(b, tx.Commit())
+
+	opts := SemanticSearchOptions{
+		QueryVector:   query,
+		Provider:      "benchmark",
+		Model:         "dense-1536",
+		InputVersion:  EmbeddingInputVersion,
+		Dimensions:    dimensions,
+		VectorBackend: "exact",
+		GuildIDs:      []string{"g1"},
+		Limit:         limit,
+	}
+	results, err := s.SearchMessagesSemantic(ctx, opts)
+	require.NoError(b, err)
+	require.Len(b, results, limit)
+	require.Equal(b, "m0", results[0].MessageID)
+
+	b.ReportAllocs()
+	for b.Loop() {
+		results, err = s.SearchMessagesSemantic(ctx, opts)
+		if err != nil {
+			b.Fatal(err)
+		}
+		if len(results) != limit || results[0].MessageID != "m0" {
+			b.Fatal("unexpected semantic search results")
+		}
+	}
+}

internal/store/store_test.go

@@ -1042,6 +1042,43 @@ func TestSearchMessagesSemanticErrors(t *testing.T) {
 	require.ErrorContains(t, err, "turbovec dimensions must be a positive multiple of 8")
 }
 
+func TestSearchMessagesSemanticZeroStoredVector(t *testing.T) {
+	t.Parallel()
+
+	for _, backend := range []string{"exact", "turbovec"} {
+		t.Run(backend, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db"))
+			require.NoError(t, err)
+			defer func() { _ = s.Close() }()
+
+			require.NoError(t, s.UpsertMessage(ctx, MessageRecord{
+				ID:                "zero-vector",
+				GuildID:           "g1",
+				ChannelID:         "c1",
+				CreatedAt:         "2026-09-27T12:00:00Z",
+				Content:           "corrupt embedding",
+				NormalizedContent: "corrupt embedding",
+				RawJSON:           `{}`,
+			}))
+			require.NoError(t, insertTestEmbedding(ctx, s, "zero-vector", "ollama", "nomic-embed-text", make([]float32, 8)))
+
+			_, err = s.SearchMessagesSemantic(ctx, SemanticSearchOptions{
+				QueryVector:   []float32{1, 0, 0, 0, 0, 0, 0, 0},
+				Provider:      "ollama",
+				Model:         "nomic-embed-text",
+				InputVersion:  EmbeddingInputVersion,
+				Dimensions:    8,
+				VectorBackend: backend,
+				Limit:         20,
+			})
+			require.EqualError(t, err, "score embedding for message zero-vector: stored embedding vector is zero")
+		})
+	}
+}
+
 func TestSearchMessagesHybridFusesAndDeduplicates(t *testing.T) {
 	t.Parallel()