diff --git a/.gitignore b/.gitignore index 59c69cd..d9995e7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ vendor/ wp-agentic-admin.zip +agentic-admin.zip *.pdf tests/e2e/screenshots/ tests/e2e/RESULTS.md diff --git a/docs/RAG-CODEBASE-SEARCH.md b/docs/RAG-CODEBASE-SEARCH.md index d6cc9f3..8acd551 100644 --- a/docs/RAG-CODEBASE-SEARCH.md +++ b/docs/RAG-CODEBASE-SEARCH.md @@ -8,7 +8,7 @@ The RAG (Retrieval-Augmented Generation) system lets the LLM answer questions ab 1. **Extracting** code from your active theme and plugins (PHP backend) 2. **Embedding** code chunks into vectors using Transformers.js (CPU/WASM) -3. **Storing** vectors in a Voy search index, persisted in IndexedDB +3. **Storing** vectors in IndexedDB, as plain `Float32Array`s 4. **Searching** with semantic similarity when users ask about code All processing happens locally — no code leaves the browser. @@ -22,7 +22,7 @@ User: "find the login function" ↓ [Transformers.js] → embed query on CPU/WASM ↓ -[Voy index] → nearest neighbor search across 1000+ code chunks +[vector index] → cosine similarity scan across 1000+ code chunks ↓ [Results] → top 3 matching code snippets with file paths + line numbers ↓ @@ -43,9 +43,16 @@ User: "find the login function" | Dependency | How loaded | Size | Purpose | |------------|-----------|------|---------| | [Transformers.js v3](https://huggingface.co/docs/transformers.js) | CDN (lazy, on first use) | ~100MB + 23MB model | Text embeddings | -| [voy-search](https://github.com/tantaraio/voy) | Bundled via npm | ~168KB WASM | Vector nearest-neighbor search | | IndexedDB | Browser native | — | Persist index across sessions | +**Why no vector-search library?** Search is an exhaustive cosine scan written in +plain JavaScript in `vector-store.js`. The embedding model emits L2-normalised +vectors, so cosine similarity is a dot product; at 384 dimensions a few thousand +chunks score in single-digit milliseconds, far below the cost of embedding the +query itself. An ANN index buys nothing at this scale, and dropping it removed +the only WebAssembly binary the plugin distributed — every shipped file now has +readable source in this repository. + **Why CDN for Transformers.js?** At ~100MB it would 17x the current 5.8MB bundle. Lazy-loading from CDN means zero cost until RAG is actually used, and the model is cached by the browser after first download. **Why CPU for embeddings?** The LLM already uses ~1.5GB VRAM via WebGPU. Running embeddings on GPU too would risk OOM. WASM/CPU is slower but avoids contention entirely. @@ -68,14 +75,14 @@ User: "index codebase" ``` User: "search code for authentication" -→ Embeds query, searches Voy index +→ Embeds query, scores it against every indexed vector → Returns top 3 matching code snippets → LLM summarizes the results ``` ### The index persists -After indexing once, the Voy index is restored from IndexedDB on page reload. No need to re-index unless your code changes. +After indexing once, the index is restored from IndexedDB on page reload. No need to re-index unless your code changes. To rebuild: say **"reindex the codebase"**. @@ -120,7 +127,7 @@ Returns 50 files per page. The JS ability paginates automatically until `has_mor ### Persistence -- Serialized Voy index stored in IndexedDB (`wp-agentic-rag-db`) +- Vectors stored as `Float32Array`s in IndexedDB (`wp-agentic-rag-db`, store `embedding-index`) - Chunk metadata (path, lines, content, type) stored alongside - Restored automatically on `vectorStore.init()` diff --git a/package-lock.json b/package-lock.json index 76abd79..1fc2f4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,7 @@ "@huggingface/transformers": "^3.8.1", "@mlc-ai/web-llm": "^0.2.82", "@wordpress/icons": "^12.0.0", - "fastest-levenshtein": "^1.0.16", - "voy-search": "^0.6.3" + "fastest-levenshtein": "^1.0.16" }, "devDependencies": { "@wordpress/hooks": "^4.46.0", @@ -22589,12 +22588,6 @@ "node": ">= 0.8" } }, - "node_modules/voy-search": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/voy-search/-/voy-search-0.6.3.tgz", - "integrity": "sha512-GRwrXcT3Qmzr/CuwpwX55XWpgqM2hUqLipSwI8bGcfsDTJGa+mFxsOXzWHNMRpcYd+U2RP73f2USLDWQu5yFdQ==", - "license": "MIT OR Apache 2.0" - }, "node_modules/w3c-xmlserializer": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", diff --git a/package.json b/package.json index 27c1415..1d61c53 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,8 @@ "test:abilities": "node tests/abilities/runner.js", "test:e2e": "node tests/abilities/e2e-runner.js", "watch": "wp-scripts start src/extensions/index.js --output-path=build-extensions", - "clean": "del-cli build-extensions wp-agentic-admin.zip", - "dist": "npm run clean && npm run build && rm -f wp-agentic-admin.zip && zip -r wp-agentic-admin.zip -@ < .distpackage", + "clean": "del-cli build-extensions agentic-admin.zip wp-agentic-admin.zip", + "dist": "npm run clean && npm run build && zip -r agentic-admin.zip -@ < .distpackage", "test": "wp-scripts test-unit-js --testMatch='**/__tests__/**/*.test.js'", "test:watch": "npm run test -- --watch", "lint:js": "wp-scripts lint-js src/", @@ -33,8 +33,7 @@ "@huggingface/transformers": "^3.8.1", "@mlc-ai/web-llm": "^0.2.82", "@wordpress/icons": "^12.0.0", - "fastest-levenshtein": "^1.0.16", - "voy-search": "^0.6.3" + "fastest-levenshtein": "^1.0.16" }, "keywords": [ "wordpress", diff --git a/readme.txt b/readme.txt index 558df4b..ea13fd6 100644 --- a/readme.txt +++ b/readme.txt @@ -83,7 +83,7 @@ Agentic Admin is fully open source under GPL-2.0-or-later. The complete, human-r https://github.com/pluginslab/wp-agentic-admin -The files under `build-extensions/` are generated from the sources in `src/` with @wordpress/scripts (webpack). They include the JavaScript bundles (`index.js`, `sw.js`, `indexing-worker.js`, and code-split chunks), the CSS, and the voy-search vector-index WebAssembly module (`*.module.wasm`, built from its npm package; source: https://github.com/tantaraio/voy). To regenerate them from a checkout: +The files under `build-extensions/` are generated from the sources in `src/` with @wordpress/scripts (webpack). They contain only JavaScript and CSS: the bundles (`index.js`, `sw.js`, `indexing-worker.js`, and code-split chunks) and the stylesheets. No WebAssembly, binaries, or other compiled artifacts are distributed with the plugin. To regenerate them from a checkout: 1. `npm install` 2. `npm run build` @@ -107,6 +107,9 @@ There is no build step for the PHP. The WebLLM engine is bundled into the plugin * Improved: ChatInput keyboard handling simplified (Space inserts a space, no push-to-talk hijacking). * Improved: KB embedding moved to a Web Worker with persistent progress across tab switches. * Pinned: Transformers.js CDN URL to @3.8.1 (was floating @3 range), privacy-first plugin shouldn't depend on a CDN range that can ship new code without a deliberate bump. +* Removed: the voy-search dependency and the WebAssembly module it distributed. Vector search is now plain JavaScript (exhaustive cosine over L2-normalised embeddings), so the plugin ships no compiled binaries at all and every distributed file has readable source in the repository. +* Fixed: re-running the knowledge base index no longer leaves vectors and chunk metadata misaligned, which could return the wrong code chunk for a query. +* Fixed: `.well-known` scanning resolves via get_home_path() instead of ABSPATH, so subdirectory installs scan the real site root. * Removed: 7 stale tab references and 6+ stale docs files (FEEDBACK-DEV.md). * Tests: 96 unit tests passing, plus the new manifest test suite (7 cases), index test suite (6 cases), knowledge-base test suite (16 cases), and react-agent regression tests (3 cases for the per-call state cleanup fix). diff --git a/src/extensions/abilities/code-search.js b/src/extensions/abilities/code-search.js index b0e89f7..61d3a5a 100644 --- a/src/extensions/abilities/code-search.js +++ b/src/extensions/abilities/code-search.js @@ -96,7 +96,7 @@ export function registerCodeSearch() { execute: async ( params ) => { try { - // Initialize vector store (loads Voy, restores from IndexedDB). + // Initialize vector store (restores index from IndexedDB). await vectorStore.init(); if ( ! vectorStore.isReady() ) { diff --git a/src/extensions/services/indexing-worker.js b/src/extensions/services/indexing-worker.js index 22cedd5..68dd567 100644 --- a/src/extensions/services/indexing-worker.js +++ b/src/extensions/services/indexing-worker.js @@ -3,7 +3,7 @@ * * Runs Transformers.js embedding inside a Web Worker to avoid blocking * the main thread during knowledge base builds. Only handles the slow - * part (neural network inference); the main thread builds the Voy index + * part (neural network inference); the main thread builds the index * and persists to IndexedDB. * * Message protocol: diff --git a/src/extensions/services/knowledge-base.js b/src/extensions/services/knowledge-base.js index 616778e..fd16c1b 100644 --- a/src/extensions/services/knowledge-base.js +++ b/src/extensions/services/knowledge-base.js @@ -106,7 +106,7 @@ function clearKBStatus() { /* ── Worker helper ───────────────────────────────────────────────────── */ /** - * Embed chunks in a Web Worker, then build Voy index on main thread. + * Embed chunks in a Web Worker, then build the index on the main thread. * * @param {Object[]} chunks Chunks to index. * @param {Function} onProgress Callback: (done, total, message) => void. @@ -133,7 +133,7 @@ function indexInWorker( chunks, onProgress ) { } else if ( msg.type === 'complete' ) { worker.terminate(); - // Build Voy index on main thread (fast, < 1s). + // Build the index on the main thread (fast, < 1s). try { onProgress( msg.chunkMetadata.length, diff --git a/src/extensions/services/vector-store.js b/src/extensions/services/vector-store.js index 55ae86d..7e157a3 100644 --- a/src/extensions/services/vector-store.js +++ b/src/extensions/services/vector-store.js @@ -1,12 +1,19 @@ /** * Vector Store Service * - * In-browser RAG vector store using Transformers.js (CDN) + voy-search (bundled) - * + IndexedDB persistence. Embeds code chunks and enables semantic search. + * In-browser RAG vector store using Transformers.js (CDN) + a plain-JS + * similarity index + IndexedDB persistence. Embeds code chunks and enables + * semantic search. * * - Transformers.js loaded lazily from CDN (~100MB, not bundled) * - Embeddings run on CPU (WASM) to avoid GPU contention with the LLM - * - Voy index + chunk metadata persisted in IndexedDB + * - Vectors + chunk metadata persisted in IndexedDB + * + * Search is an exhaustive cosine scan. The embedding model emits L2-normalised + * vectors (`normalize: true`), so cosine similarity is a plain dot product. + * At 384 dimensions a few thousand chunks score in single-digit milliseconds, + * which is well under the embedding cost of the query itself, so an ANN index + * would buy nothing at this scale. */ import { createLogger } from '../utils/logger'; @@ -17,8 +24,10 @@ const TRANSFORMERS_CDN = 'https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1'; const MODEL_NAME = 'Xenova/all-MiniLM-L6-v2'; const DB_NAME = 'wp-agentic-rag-db'; -const DB_VERSION = 1; -const STORE_INDEX = 'voy-index'; +// v2 replaced the voy-search index store with plain Float32Array vectors. +const DB_VERSION = 2; +const STORE_INDEX = 'embedding-index'; +const LEGACY_STORE_INDEX = 'voy-index'; const STORE_CHUNKS = 'chunk-metadata'; const BATCH_SIZE = 10; @@ -28,9 +37,11 @@ const BATCH_SIZE = 10; let pipeline = null; /** - * @type {import('voy-search').Voy|null} + * Indexed vectors, positionally aligned with `chunkMetadata`. + * + * @type {Float32Array[]} */ -let voyInstance = null; +let vectors = []; /** * @type {Array} @@ -64,6 +75,11 @@ function openDB() { if ( ! db.objectStoreNames.contains( STORE_CHUNKS ) ) { db.createObjectStore( STORE_CHUNKS ); } + // Drop the old voy-search index; its serialised format is not + // readable here. Affected users re-index from the settings screen. + if ( db.objectStoreNames.contains( LEGACY_STORE_INDEX ) ) { + db.deleteObjectStore( LEGACY_STORE_INDEX ); + } }; request.onsuccess = () => resolve( request.result ); @@ -165,42 +181,67 @@ async function loadPipeline() { } /** - * Load Voy search instance from bundled npm package. + * Convert worker/index embedding entries into the internal vector array. + * + * Entries use the `{ id, title, url, embeddings }` shape produced by + * `indexing-worker.js`; only the vector itself is retained, positionally. + * + * @param {Object[]} entries Embedding entries. + * @return {Float32Array[]} Vectors. + */ +function toVectors( entries ) { + return entries.map( ( entry ) => Float32Array.from( entry.embeddings ) ); +} + +/** + * Restore the index and chunk metadata from IndexedDB. + * + * Vectors and metadata are positionally aligned, so a length mismatch (a + * partial write, or the v1 → v2 upgrade dropping the old voy index) resets + * both rather than serving results against the wrong metadata. * * @return {Promise} */ -async function loadVoy() { - if ( voyInstance ) { +async function loadIndex() { + if ( vectors.length ) { return; } - const { Voy } = await import( 'voy-search' ); - - // Try to restore from IndexedDB. try { - const savedIndex = await dbGet( STORE_INDEX, 'current' ); + const savedVectors = await dbGet( STORE_INDEX, 'current' ); const savedChunks = await dbGet( STORE_CHUNKS, 'current' ); - if ( savedIndex && savedChunks ) { - voyInstance = Voy.deserialize( savedIndex ); + if ( + Array.isArray( savedVectors ) && + Array.isArray( savedChunks ) && + savedVectors.length === savedChunks.length && + savedVectors.length > 0 + ) { + vectors = savedVectors.map( ( v ) => Float32Array.from( v ) ); chunkMetadata = savedChunks; log.info( `Restored index from IndexedDB (${ chunkMetadata.length } chunks).` ); return; } + + if ( savedVectors || savedChunks ) { + log.warn( + 'Persisted index is incomplete or from an older format; starting empty. Re-index to rebuild.' + ); + } } catch ( err ) { log.warn( 'Could not restore index from IndexedDB:', err.message ); } - voyInstance = new Voy( { embeddings: [] } ); + vectors = []; chunkMetadata = []; - log.info( 'Created new empty Voy index.' ); + log.info( 'Created new empty index.' ); } /** * Initialize the vector store. - * Loads Voy immediately (bundled), defers Transformers.js until needed. + * Restores any persisted index, defers Transformers.js until needed. * * @return {Promise} */ @@ -212,7 +253,7 @@ async function init() { initializing = true; try { - await loadVoy(); + await loadIndex(); initialized = true; log.info( 'Vector store initialized.' ); } catch ( err ) { @@ -244,14 +285,16 @@ async function embed( text ) { } /** - * Persist the current Voy index and chunk metadata to IndexedDB. + * Persist the current vectors and chunk metadata to IndexedDB. + * + * Float32Array survives the structured clone algorithm, so the vectors are + * stored as-is rather than being stringified. * * @return {Promise} */ async function persist() { try { - const serialized = voyInstance.serialize(); - await dbPut( STORE_INDEX, 'current', serialized ); + await dbPut( STORE_INDEX, 'current', vectors ); await dbPut( STORE_CHUNKS, 'current', chunkMetadata ); log.debug( `Persisted index (${ chunkMetadata.length } chunks).` ); } catch ( err ) { @@ -261,7 +304,7 @@ async function persist() { /** * Index an array of code chunks. - * Embeds in batches and adds to the Voy index. + * Embeds in batches and rebuilds the index. * * @param {Object[]} chunks Array of { path, start_line, end_line, content, type }. * @param {Function} [onProgress] Optional callback: (indexed, total) => void. @@ -275,10 +318,11 @@ async function index( chunks, onProgress ) { // Ensure pipeline is loaded for embedding. await loadPipeline(); - const { Voy } = await import( 'voy-search' ); - - // Reset index with fresh data. + // Reset index with fresh data. chunkMetadata is rebuilt alongside + // `embeddings` and must be cleared with it, or a second index() run leaves + // the two positionally misaligned and search returns the wrong chunks. const embeddings = []; + chunkMetadata = []; let indexed = 0; for ( let i = 0; i < chunks.length; i += BATCH_SIZE ) { @@ -323,8 +367,8 @@ async function index( chunks, onProgress ) { } } - // Build new Voy index with all embeddings. - voyInstance = new Voy( { embeddings } ); + // Build the new index from all embeddings. + vectors = toVectors( embeddings ); // Persist to IndexedDB. await persist(); @@ -333,6 +377,24 @@ async function index( chunks, onProgress ) { return indexed; } +/** + * Cosine similarity between two L2-normalised vectors. + * + * Both operands come from the embedding pipeline with `normalize: true`, so + * the magnitudes are 1 and the dot product *is* the cosine. + * + * @param {Float32Array} a First vector. + * @param {Float32Array} b Second vector. + * @return {number} Similarity in [-1, 1]. + */ +function cosine( a, b ) { + let dot = 0; + for ( let i = 0; i < a.length; i++ ) { + dot += a[ i ] * b[ i ]; + } + return dot; +} + /** * Search the vector index for relevant code chunks. * @@ -341,30 +403,36 @@ async function index( chunks, onProgress ) { * @return {Promise} Array of { path, start_line, end_line, content, type, score }. */ async function search( query, topK = 3 ) { - if ( ! initialized || ! voyInstance || chunkMetadata.length === 0 ) { + if ( ! initialized || vectors.length === 0 || chunkMetadata.length === 0 ) { return []; } - const queryEmbedding = await embed( query ); - // Voy.search() requires Float32Array, not a plain Array. - const queryFloat32 = new Float32Array( queryEmbedding ); - const results = voyInstance.search( queryFloat32, topK ); + const queryVector = Float32Array.from( await embed( query ) ); + + const ranked = vectors + .map( ( vector, idx ) => ( { + idx, + score: cosine( queryVector, vector ), + } ) ) + .sort( ( a, b ) => b.score - a.score ) + .slice( 0, topK ); log.info( - 'Voy results:', - results.neighbors.map( ( n ) => `${ n.id }: ${ n.title }` ) + 'Search results:', + ranked.map( + ( r ) => + `${ r.idx }: ${ + chunkMetadata[ r.idx ]?.path + } (${ r.score.toFixed( 3 ) })` + ) ); - return results.neighbors.map( ( neighbor, rank ) => { - const idx = parseInt( neighbor.id, 10 ); - const meta = chunkMetadata[ idx ]; - return { - ...meta, - // Voy doesn't return distances; results are pre-sorted by similarity. - // Use inverse rank as a rough relevance indicator. - score: topK - rank, - }; - } ); + return ranked.map( ( { idx, score } ) => ( { + ...chunkMetadata[ idx ], + // Real cosine similarity, unlike the inverse-rank placeholder the + // previous ANN index forced (it returned no distances). + score: Number( score.toFixed( 3 ) ), + } ) ); } /** @@ -386,17 +454,17 @@ function getChunkCount() { } /** - * Reload the Voy index and chunk metadata from IndexedDB. + * Reload the index and chunk metadata from IndexedDB. * Used after the indexing worker finishes persisting new data. * * @return {Promise} */ async function reload() { - voyInstance = null; + vectors = []; chunkMetadata = []; initialized = false; initializing = false; - await loadVoy(); + await loadIndex(); initialized = true; log.info( `Reloaded index from IndexedDB (${ chunkMetadata.length } chunks).` @@ -404,10 +472,10 @@ async function reload() { } /** - * Build the Voy index from pre-computed embeddings and persist. + * Build the index from pre-computed embeddings and persist. * Used after the indexing worker returns embeddings from a background thread. * - * @param {Object[]} embeddings Pre-computed Voy embedding entries. + * @param {Object[]} embeddings Pre-computed embedding entries. * @param {Object[]} metadata Chunk metadata array. * @return {Promise} Number of chunks indexed. */ @@ -416,8 +484,7 @@ async function buildFromEmbeddings( embeddings, metadata ) { await init(); } - const { Voy } = await import( 'voy-search' ); - voyInstance = new Voy( { embeddings } ); + vectors = toVectors( embeddings ); chunkMetadata = metadata; await persist(); @@ -434,8 +501,7 @@ async function buildFromEmbeddings( embeddings, metadata ) { * @return {Promise} */ async function clear() { - const { Voy } = await import( 'voy-search' ); - voyInstance = new Voy( { embeddings: [] } ); + vectors = []; chunkMetadata = []; await persist(); log.info( 'Vector store cleared.' ); diff --git a/webpack.config.js b/webpack.config.js index 4b3ec88..c2d8c4d 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -13,12 +13,6 @@ const path = require( 'path' ); module.exports = { ...defaultConfig, - // Enable WASM support for voy-search vector database. - experiments: { - ...( defaultConfig.experiments || {} ), - asyncWebAssembly: true, - }, - entry: { // Main application bundle index: path.resolve( __dirname, 'src/extensions/index.js' ), @@ -38,7 +32,7 @@ module.exports = { // and src/extensions/components/VoiceButton.jsx for v1.4 (per roadmap). // Re-add this entry to ship voice input again. - // Indexing Web Worker - background embedding + Voy indexing + // Indexing Web Worker - background embedding 'indexing-worker': { import: path.resolve( __dirname,