diff --git a/README.md b/README.md index 576ed5ae..38b09bd7 100644 --- a/README.md +++ b/README.md @@ -241,34 +241,50 @@ The large default context can exceed memory for big models on constrained hardwa ### Embedding Model -`skilld search` uses a local embedding model. It runs offline through transformers.js. It needs no API key or network after the first download. Pick one under **Embedding model** in `skilld config`: +`skilld search` is powered by a local embedding model. It runs offline through transformers.js. It needs no API key or network traffic after the first download. Pick one under **Embedding model** in `skilld config`: | Model | Dimensions | Notes | |-------|-----------:|-------| | `bge-small-en-v1.5` | 384 | Default. Fastest to index, smallest download. | | `bge-base-en-v1.5` | 768 | Balanced accuracy and speed. | +| `Xenova/bge-large-en-v1.5` | 1024 | Most accurate English retrieval, slowest to index. | | `bge-m3` | 1024 | Multilingual, 8192-token context. | -Larger models retrieve more accurately but cost more time and memory when indexing. Set `SKILLD_EMBED_MODEL` to override the saved setting: +Larger models retrieve more accurately but cost more time and memory when indexing. Locally-pulled [Ollama](#ollama-embedding-models) models can be used too. Export `SKILLD_EMBED_MODEL` to override the saved setting for every index and search command in the current shell: ```bash export SKILLD_EMBED_MODEL=bge-m3 -skilld update --force +skilld update ``` -Each search index belongs to one model and device. Keep environment overrides set for both indexing and querying. Rebuild indexes after either setting changes: +Search indexes must use one embedding model and device. Rebuild them after switching either setting: ```bash -skilld update --force +skilld update ``` +### Ollama Embedding Models + +If [Ollama](https://ollama.com) is running, locally-pulled embedding models appear in the **Embedding model** picker alongside the built-in ones. They are addressed as `ollama:`, matching the `-m ollama:` syntax used for enhancement models: + +```bash +ollama pull qwen3-embedding +SKILLD_EMBED_MODEL=ollama:qwen3-embedding skilld add npm:vue +``` + +Only models that advertise the `embedding` capability are listed, so chat models cannot be selected by mistake. Dimensions and context length are read from Ollama. skilld validates each vector before indexing. + +This talks to Ollama's HTTP API directly. It needs no additional dependency or API key. Set `OLLAMA_HOST` to point at a non-default daemon. If Ollama is not running, the picker simply shows the built-in models. + +Ollama manages its own execution device, so **Embedding device** does not apply to `ollama:` models. + ### Embedding Device The embedding model runs on the CPU by default. **Embedding device** in `skilld config` moves it onto a GPU backend, which can be substantially faster: | Device | Notes | |--------|-------| -| `auto` | Default. Lets transformers.js choose, CPU under Node. | +| `auto` | Default. Lets transformers.js choose; CPU under Node. | | `cpu` | Always available, predictable. | | `webgpu` | Fastest on Apple Silicon in testing. | | `coreml` | Apple Neural Engine. Measured slower than CPU for these models. | @@ -281,13 +297,13 @@ Measured on an Apple M5 Max, 120 documents, best of 3 after warm-up (docs/sec): | `bge-base-en-v1.5` | 198 | 68 | **580** | | `Xenova/bge-large-en-v1.5` | 71 | 9 | **201** | -WebGPU was 2.6 to 2.9 times faster than CPU at every size. `bge-large` on WebGPU indexed faster than `bge-base` on CPU. CoreML was consistently slower. +WebGPU was 2.6 to 2.9 times faster than CPU at every size. On WebGPU, `bge-large` indexed faster than `bge-base` on CPU and improved retrieval. CoreML was consistently slower. -The ranking is hardware-specific, so benchmark before trusting a device on other machines. Set `SKILLD_EMBED_DEVICE` to override the saved setting: +The ranking is hardware-specific, so benchmark before trusting a device on other machines. Export `SKILLD_EMBED_DEVICE` to override the saved setting in the current shell: ```bash export SKILLD_EMBED_DEVICE=cpu -skilld update --force +skilld update ``` If a backend is unavailable, indexing fails to start. Switch back to `auto`. diff --git a/package.json b/package.json index 2f59b151..3dffe712 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "devDependencies": { "@antfu/eslint-config": "catalog:dev-lint", "@types/node": "catalog:dev-build", + "@typescript/native": "catalog:", "@vitest/coverage-v8": "catalog:dev-test", "bumpp": "catalog:", "eslint": "catalog:dev-lint", diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 061a82bc..de2eec79 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -59,6 +59,7 @@ "@antfu/eslint-config": "catalog:dev-lint", "@arethetypeswrong/cli": "catalog:", "@types/node": "catalog:dev-build", + "@typescript/native": "catalog:", "eslint": "catalog:dev-lint", "obuild": "catalog:dev-build", "publint": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cb923ee9..360b9031 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -18,6 +18,9 @@ catalogs: '@napi-rs/keyring': specifier: ^1.3.0 version: 1.3.0 + '@typescript/native': + specifier: npm:typescript@7.0.2 + version: 7.0.2 bumpp: specifier: ^12.2.0 version: 12.2.0 @@ -55,8 +58,8 @@ catalogs: specifier: ^1.3.10 version: 1.3.10 typescript: - specifier: 7.0.2 - version: 7.0.2 + specifier: npm:@typescript/typescript6@^6.0.2 + version: 6.0.2 unagent: specifier: ^0.0.8 version: 0.0.8 @@ -167,7 +170,7 @@ importers: version: 2.0.3 retriv: specifier: 'catalog:' - version: 0.15.0(@huggingface/transformers@4.2.0)(sqlite-vec@0.1.9)(typescript@7.0.2) + version: 0.15.0(@huggingface/transformers@4.2.0)(@typescript/typescript6@6.0.2)(sqlite-vec@0.1.9) skilld-protocol: specifier: workspace:* version: link:packages/protocol @@ -182,7 +185,7 @@ importers: version: 1.3.10 typescript: specifier: 'catalog:' - version: 7.0.2 + version: '@typescript/typescript6@6.0.2' unagent: specifier: 'catalog:' version: 0.0.8(@huggingface/transformers@4.2.0)(sqlite-vec@0.1.9) @@ -192,10 +195,13 @@ importers: devDependencies: '@antfu/eslint-config': specifier: catalog:dev-lint - version: 9.2.0(@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2))(@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)(vitest@4.1.10) + version: 9.2.0(@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0))(@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(vitest@4.1.10) '@types/node': specifier: catalog:dev-build version: 26.1.2 + '@typescript/native': + specifier: 'catalog:' + version: typescript@7.0.2 '@vitest/coverage-v8': specifier: catalog:dev-test version: 4.1.10(vitest@4.1.10) @@ -207,7 +213,7 @@ importers: version: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) obuild: specifier: catalog:dev-build - version: 0.4.38(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4)(typescript@7.0.2) + version: 0.4.38(@typescript/typescript6@6.0.2)(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4) tsx: specifier: 'catalog:' version: 4.23.8 @@ -227,25 +233,28 @@ importers: devDependencies: '@antfu/eslint-config': specifier: catalog:dev-lint - version: 9.2.0(@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2))(@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)(vitest@4.1.10) + version: 9.2.0(@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0))(@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(vitest@4.1.10) '@arethetypeswrong/cli': specifier: 'catalog:' version: 0.18.5 '@types/node': specifier: catalog:dev-build version: 26.1.2 + '@typescript/native': + specifier: 'catalog:' + version: typescript@7.0.2 eslint: specifier: catalog:dev-lint version: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) obuild: specifier: catalog:dev-build - version: 0.4.38(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4)(typescript@7.0.2) + version: 0.4.38(@typescript/typescript6@6.0.2)(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4) publint: specifier: 'catalog:' version: 0.3.23 typescript: specifier: 'catalog:' - version: 7.0.2 + version: '@typescript/typescript6@6.0.2' vitest: specifier: catalog:dev-test version: 4.1.10(@opentelemetry/api@1.9.0)(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(vite@7.3.1(@types/node@26.1.2)(jiti@2.7.0)(tsx@4.23.8)(yaml@2.9.0)) @@ -1956,6 +1965,10 @@ packages: cpu: [x64] os: [win32] + '@typescript/typescript6@6.0.2': + resolution: {integrity: sha512-mbCddXd+jm7hfx7w2YU64/Av4/NqqeG3GoRZgxPcgoTxYjhrcfJRw9ULch71SS4G+Q3bOXFhRvPqjguN0Hyp5w==} + hasBin: true + '@vitest/coverage-v8@4.1.10': resolution: {integrity: sha512-IM49HmthevbgAO4anp1hwtoT9wYe59w0LR00gr+eagHE+ZJ5lK4sLPeO0ubgoJcwLk6dehU3R24N+FbEEKDc8g==} peerDependencies: @@ -3817,6 +3830,11 @@ packages: engines: {node: '>=14.17'} hasBin: true + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + typescript@7.0.2: resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} engines: {node: '>=16.20.0'} @@ -4126,7 +4144,7 @@ snapshots: '@andrewbranch/untar.js@1.0.4': {} - '@antfu/eslint-config@9.2.0(@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2))(@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)(vitest@4.1.10)': + '@antfu/eslint-config@9.2.0(@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0))(@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(vitest@4.1.10)': dependencies: '@antfu/install-pkg': 1.1.0 '@clack/prompts': 1.7.0 @@ -4134,9 +4152,9 @@ snapshots: '@eslint-community/eslint-plugin-eslint-comments': 4.7.2(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) '@eslint/markdown': 8.0.3(supports-color@7.2.0) '@stylistic/eslint-plugin': 5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) - '@typescript-eslint/parser': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) - '@vitest/eslint-plugin': 1.6.26(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)(vitest@4.1.10) + '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + '@typescript-eslint/parser': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + '@vitest/eslint-plugin': 1.6.26(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(vitest@4.1.10) ansis: 4.3.1 cac: 7.0.0 eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) @@ -4144,19 +4162,19 @@ snapshots: eslint-flat-config-utils: 3.2.0 eslint-merge-processors: 2.0.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-antfu: 3.2.3(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - eslint-plugin-command: 3.5.3(@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2))(@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-command: 3.5.3(@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0))(@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-import-lite: 0.6.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-jsdoc: 63.3.3(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint-plugin-jsonc: 3.4.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - eslint-plugin-n: 18.2.2(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(typescript@7.0.2) + eslint-plugin-n: 18.2.2(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-no-only-tests: 3.4.0 - eslint-plugin-perfectionist: 5.10.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + eslint-plugin-perfectionist: 5.10.1(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint-plugin-pnpm: 1.7.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-regexp: 3.1.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-plugin-toml: 1.5.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint-plugin-unicorn: 72.0.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - eslint-plugin-unused-imports: 4.4.1(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - eslint-plugin-vue: 10.10.0(@stylistic/eslint-plugin@5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)))(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)) + eslint-plugin-unused-imports: 4.4.1(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) + eslint-plugin-vue: 10.10.0(@stylistic/eslint-plugin@5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)))(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)) eslint-plugin-yml: 3.8.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint-processor-vue-blocks: 2.0.0(@vue/compiler-sfc@3.5.27)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) globals: 17.9.0 @@ -5325,40 +5343,40 @@ snapshots: '@types/unist@3.0.3': {} - '@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@eslint-community/regexpp': 4.12.2 - '@typescript-eslint/parser': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/parser': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) '@typescript-eslint/scope-manager': 8.66.0 - '@typescript-eslint/type-utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) - '@typescript-eslint/utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/type-utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + '@typescript-eslint/utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) '@typescript-eslint/visitor-keys': 8.66.0 eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) ignore: 7.0.6 natural-compare: 1.4.0 - ts-api-utils: 2.5.0(typescript@7.0.2) - typescript: 7.0.2 + ts-api-utils: 2.5.0(@typescript/typescript6@6.0.2) + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@typescript-eslint/scope-manager': 8.66.0 '@typescript-eslint/types': 8.66.0 - '@typescript-eslint/typescript-estree': 8.66.0(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/typescript-estree': 8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0) '@typescript-eslint/visitor-keys': 8.66.0 debug: 4.4.3(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color - '@typescript-eslint/project-service@8.66.0(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/project-service@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0)': dependencies: - '@typescript-eslint/tsconfig-utils': 8.66.0(typescript@7.0.2) + '@typescript-eslint/tsconfig-utils': 8.66.0(@typescript/typescript6@6.0.2) '@typescript-eslint/types': 8.66.0 debug: 4.4.3(supports-color@7.2.0) - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color @@ -5367,47 +5385,47 @@ snapshots: '@typescript-eslint/types': 8.66.0 '@typescript-eslint/visitor-keys': 8.66.0 - '@typescript-eslint/tsconfig-utils@8.66.0(typescript@7.0.2)': + '@typescript-eslint/tsconfig-utils@8.66.0(@typescript/typescript6@6.0.2)': dependencies: - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' - '@typescript-eslint/type-utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/type-utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@typescript-eslint/types': 8.66.0 - '@typescript-eslint/typescript-estree': 8.66.0(supports-color@7.2.0)(typescript@7.0.2) - '@typescript-eslint/utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/typescript-estree': 8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0) + '@typescript-eslint/utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) debug: 4.4.3(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) - ts-api-utils: 2.5.0(typescript@7.0.2) - typescript: 7.0.2 + ts-api-utils: 2.5.0(@typescript/typescript6@6.0.2) + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color '@typescript-eslint/types@8.66.0': {} - '@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0)': dependencies: - '@typescript-eslint/project-service': 8.66.0(supports-color@7.2.0)(typescript@7.0.2) - '@typescript-eslint/tsconfig-utils': 8.66.0(typescript@7.0.2) + '@typescript-eslint/project-service': 8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0) + '@typescript-eslint/tsconfig-utils': 8.66.0(@typescript/typescript6@6.0.2) '@typescript-eslint/types': 8.66.0 '@typescript-eslint/visitor-keys': 8.66.0 debug: 4.4.3(supports-color@7.2.0) minimatch: 10.2.6 semver: 7.8.5 tinyglobby: 0.2.17 - ts-api-utils: 2.5.0(typescript@7.0.2) - typescript: 7.0.2 + ts-api-utils: 2.5.0(@typescript/typescript6@6.0.2) + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)': + '@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)': dependencies: '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) '@typescript-eslint/scope-manager': 8.66.0 '@typescript-eslint/types': 8.66.0 - '@typescript-eslint/typescript-estree': 8.66.0(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/typescript-estree': 8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - supports-color @@ -5476,6 +5494,10 @@ snapshots: '@typescript/typescript-win32-x64@7.0.2': optional: true + '@typescript/typescript6@6.0.2': + dependencies: + '@typescript/old': typescript@6.0.3 + '@vitest/coverage-v8@4.1.10(vitest@4.1.10)': dependencies: '@bcoe/v8-coverage': 1.0.2 @@ -5490,14 +5512,14 @@ snapshots: tinyrainbow: 3.1.1 vitest: 4.1.10(@opentelemetry/api@1.9.0)(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(vite@7.3.1(@types/node@26.1.2)(jiti@2.7.0)(tsx@4.23.8)(yaml@2.9.0)) - '@vitest/eslint-plugin@1.6.26(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2)(vitest@4.1.10)': + '@vitest/eslint-plugin@1.6.26(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(vitest@4.1.10)': dependencies: '@typescript-eslint/scope-manager': 8.66.0 - '@typescript-eslint/utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) optionalDependencies: - '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) - typescript: 7.0.2 + '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) + typescript: '@typescript/typescript6@6.0.2' vitest: 4.1.10(@opentelemetry/api@1.9.0)(@types/node@26.1.2)(@vitest/coverage-v8@4.1.10)(vite@7.3.1(@types/node@26.1.2)(jiti@2.7.0)(tsx@4.23.8)(yaml@2.9.0)) transitivePeerDependencies: - supports-color @@ -6028,11 +6050,11 @@ snapshots: dependencies: eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) - eslint-plugin-command@3.5.3(@typescript-eslint/typescript-estree@8.66.0(supports-color@7.2.0)(typescript@7.0.2))(@typescript-eslint/utils@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): + eslint-plugin-command@3.5.3(@typescript-eslint/typescript-estree@8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0))(@typescript-eslint/utils@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): dependencies: '@es-joy/jsdoccomment': 0.88.0 - '@typescript-eslint/typescript-estree': 8.66.0(supports-color@7.2.0)(typescript@7.0.2) - '@typescript-eslint/utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/typescript-estree': 8.66.0(@typescript/typescript6@6.0.2)(supports-color@7.2.0) + '@typescript-eslint/utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) eslint-plugin-es-x@7.8.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): @@ -6081,7 +6103,7 @@ snapshots: transitivePeerDependencies: - '@eslint/json' - eslint-plugin-n@18.2.2(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(typescript@7.0.2): + eslint-plugin-n@18.2.2(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): dependencies: '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) enhanced-resolve: 5.24.5 @@ -6093,13 +6115,13 @@ snapshots: ignore: 5.3.2 semver: 7.8.5 optionalDependencies: - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' eslint-plugin-no-only-tests@3.4.0: {} - eslint-plugin-perfectionist@5.10.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2): + eslint-plugin-perfectionist@5.10.1(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0): dependencies: - '@typescript-eslint/utils': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/utils': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) natural-orderby: 5.0.0 transitivePeerDependencies: @@ -6163,13 +6185,13 @@ snapshots: strip-indent: 4.1.1 yaml: 2.9.0 - eslint-plugin-unused-imports@4.4.1(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): + eslint-plugin-unused-imports@4.4.1(@typescript-eslint/eslint-plugin@8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): dependencies: eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) optionalDependencies: - '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/eslint-plugin': 8.66.0(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) - eslint-plugin-vue@10.10.0(@stylistic/eslint-plugin@5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)))(@typescript-eslint/parser@8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)): + eslint-plugin-vue@10.10.0(@stylistic/eslint-plugin@5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)))(@typescript-eslint/parser@8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0))(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(vue-eslint-parser@10.4.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)): dependencies: '@eslint-community/eslint-utils': 4.10.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) eslint: 10.8.0(jiti@2.7.0)(supports-color@7.2.0) @@ -6181,7 +6203,7 @@ snapshots: xml-name-validator: 5.0.0 optionalDependencies: '@stylistic/eslint-plugin': 5.10.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)) - '@typescript-eslint/parser': 8.66.0(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0)(typescript@7.0.2) + '@typescript-eslint/parser': 8.66.0(@typescript/typescript6@6.0.2)(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0))(supports-color@7.2.0) eslint-plugin-yml@3.8.1(eslint@10.8.0(jiti@2.7.0)(supports-color@7.2.0)): dependencies: @@ -7040,7 +7062,7 @@ snapshots: obug@2.1.4: {} - obuild@0.4.38(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4)(typescript@7.0.2): + obuild@0.4.38(@typescript/typescript6@6.0.2)(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4): dependencies: c12: 4.0.0-beta.5(chokidar@5.0.0)(dotenv@17.4.2)(giget@3.3.1)(jiti@2.7.0)(magicast@0.5.4) consola: 3.4.2 @@ -7049,7 +7071,7 @@ snapshots: magic-string: 0.30.21 pathe: 2.0.3 rolldown: 1.2.3 - rolldown-plugin-dts: 0.27.14(rolldown@1.2.3)(typescript@7.0.2) + rolldown-plugin-dts: 0.27.14(@typescript/typescript6@6.0.2)(rolldown@1.2.3) tinyglobby: 0.2.17 transitivePeerDependencies: - '@typescript/native-preview' @@ -7280,15 +7302,15 @@ snapshots: onetime: 7.0.0 signal-exit: 4.1.0 - retriv@0.15.0(@huggingface/transformers@4.2.0)(sqlite-vec@0.1.9)(typescript@7.0.2): + retriv@0.15.0(@huggingface/transformers@4.2.0)(@typescript/typescript6@6.0.2)(sqlite-vec@0.1.9): optionalDependencies: '@huggingface/transformers': 4.2.0 sqlite-vec: 0.1.9 - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' retry@0.13.1: {} - rolldown-plugin-dts@0.27.14(rolldown@1.2.3)(typescript@7.0.2): + rolldown-plugin-dts@0.27.14(@typescript/typescript6@6.0.2)(rolldown@1.2.3): dependencies: dts-resolver: 3.0.0 get-tsconfig: 5.0.0-beta.5 @@ -7298,7 +7320,7 @@ snapshots: yuku-codegen: 0.8.3 yuku-parser: 0.8.3 optionalDependencies: - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' transitivePeerDependencies: - oxc-resolver @@ -7545,9 +7567,9 @@ snapshots: ts-algebra@2.0.0: {} - ts-api-utils@2.5.0(typescript@7.0.2): + ts-api-utils@2.5.0(@typescript/typescript6@6.0.2): dependencies: - typescript: 7.0.2 + typescript: '@typescript/typescript6@6.0.2' tslib@2.8.1: {} @@ -7571,6 +7593,8 @@ snapshots: typescript@5.6.1-rc: {} + typescript@6.0.3: {} + typescript@7.0.2: optionalDependencies: '@typescript/typescript-aix-ppc64': 7.0.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 98298b4e..6e5051b1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -16,6 +16,7 @@ catalog: '@earendil-works/pi-ai': ^0.83.0 '@mdream/crawl': ^1.5.12 '@napi-rs/keyring': ^1.3.0 + '@typescript/native': npm:typescript@7.0.2 bumpp: ^12.2.0 giget: ^3.3.1 hookable: ^6.1.1 @@ -28,7 +29,7 @@ catalog: std-env: ^4.2.0 tsx: ^4.23.8 typebox: ^1.3.10 - typescript: 7.0.2 + typescript: npm:@typescript/typescript6@^6.0.2 unagent: ^0.0.8 verkit: ^0.3.2 zod: ^4.4.3 diff --git a/src/agent/clis/executors.ts b/src/agent/clis/executors.ts index 16e1d017..05a21f08 100644 --- a/src/agent/clis/executors.ts +++ b/src/agent/clis/executors.ts @@ -14,8 +14,8 @@ import type { OptimizeModel } from './types.ts' import { getSkillReferenceDirs } from '../../cache/index.ts' import { CLI_ADAPTERS, CLI_MODELS } from './index.ts' import { isOllamaModel, ollamaExecutor } from './ollama.ts' -import { getAvailablePiAiModels, isPiAiModel, optimizeSectionPiAi } from './pi-ai.ts' import { createPiAiModels } from './pi-ai-auth.ts' +import { getAvailablePiAiModels, isPiAiModel, optimizeSectionPiAi } from './pi-ai.ts' import { spawnCliAndStream } from './runner.ts' function cliExecutor(model: OptimizeModel): SectionExecutor | { error: string } { diff --git a/src/agent/clis/ollama.ts b/src/agent/clis/ollama.ts index 72c40f4c..e2b3c813 100644 --- a/src/agent/clis/ollama.ts +++ b/src/agent/clis/ollama.ts @@ -17,6 +17,7 @@ import type { SectionExecutor } from './runner.ts' import type { OptimizeModel } from './types.ts' +import { ollamaHost } from '../../core/ollama-host.ts' const OLLAMA_PREFIX = 'ollama:' @@ -29,15 +30,6 @@ export function parseOllamaModelId(model: string): string | null { return isOllamaModel(model) ? model.slice(OLLAMA_PREFIX.length) : null } -const HAS_SCHEME_RE = /^https?:\/\// -const TRAILING_SLASH_RE = /\/$/ - -function ollamaHost(): string { - const raw = process.env.OLLAMA_HOST || 'http://localhost:11434' - const withScheme = HAS_SCHEME_RE.test(raw) ? raw : `http://${raw}` - return withScheme.replace(TRAILING_SLASH_RE, '') -} - interface OllamaChatChunk { message?: { content?: string } done?: boolean diff --git a/src/agent/clis/pi-ai-auth.ts b/src/agent/clis/pi-ai-auth.ts index 04de33db..fa35ecc3 100644 --- a/src/agent/clis/pi-ai-auth.ts +++ b/src/agent/clis/pi-ai-auth.ts @@ -149,16 +149,20 @@ export async function loginOAuthProvider(providerId: string, callbacks: LoginCal return false const notify = (event: AuthEvent): void => { - if (event.type === 'auth_url') + if (event.type === 'auth_url') { callbacks.onAuth(event.url, event.instructions) + } else if (event.type === 'device_code') { - if (callbacks.onDeviceCode) + if (callbacks.onDeviceCode) { callbacks.onDeviceCode(event.userCode, event.verificationUri) - else + } + else { callbacks.onAuth(event.verificationUri, `Enter code ${event.userCode}`) + } } - else + else { callbacks.onProgress?.(event.message) + } } const prompt = async (input: AuthPrompt): Promise => { if (input.type !== 'select') diff --git a/src/cache/internal/references.ts b/src/cache/internal/references.ts index f5326c1e..7903c0bc 100644 --- a/src/cache/internal/references.ts +++ b/src/cache/internal/references.ts @@ -20,6 +20,7 @@ import { dirname, join } from 'pathe' import { defaultFeatures, readConfig } from '../../core/config.ts' import { getPackageDbPath, getRepoCacheDir, skillInternalDir } from '../../core/paths.ts' import { hasShippedDocs } from '../../core/prepare.ts' +import { hasIndexEmbeddingIdentity, resolveEmbeddingIdentity } from '../../retriv/index-identity.ts' import { classifyCachedDoc } from './classify.ts' import { clearCache, @@ -202,7 +203,7 @@ export function loadCachedReferences(opts: LoadCachedReferencesOptions): CachedR // Load cached docs for indexing if db doesn't exist yet const dbPath = getPackageDbPath(packageName, version) - if (!existsSync(dbPath)) { + if (!existsSync(dbPath) || !hasIndexEmbeddingIdentity(dbPath, resolveEmbeddingIdentity())) { onProgress('Reading cached docs for indexing') const cached = readCachedDocs(packageName, version) for (const doc of cached) { diff --git a/src/commands/config.ts b/src/commands/config.ts index 3f1ac873..6662cb3b 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -12,6 +12,7 @@ import { NO_MODELS_MESSAGE, OAUTH_NOTE, pickModel } from '../cli/model-picker.ts import { defaultFeatures, readConfig, updateConfig } from '../core/config.ts' import { getProjectState } from '../core/skills.ts' import { DEFAULT_EMBED_DEVICE, DEFAULT_EMBED_MODEL, EMBED_DEVICES, EMBED_MODELS, resolveEmbedModel } from '../retriv/models.ts' +import { getAvailableOllamaEmbedModels, isOllamaEmbedModel } from '../retriv/ollama-embeddings.ts' export async function configCommand(): Promise { const initConfig = readConfig() @@ -248,7 +249,7 @@ async function configureModel(): Promise { } } -// Embedding model selection +// ── Embedding model selection ──────────────────────────────────────── async function configureEmbedModel(): Promise { const config = readConfig() @@ -259,13 +260,22 @@ async function configureEmbedModel(): Promise { p.log.warn(`SKILLD_EMBED_MODEL is set to ${envOverride} and overrides this setting for the current shell.`) } + const builtIn = EMBED_MODELS.map(m => ({ + label: m.label, + value: m.id, + hint: `${m.dimensions}d · ${m.hint}`, + })) + // Locally-pulled Ollama models are additive: an unreachable daemon simply + // contributes nothing rather than blocking the picker. + const ollama = (await getAvailableOllamaEmbedModels()).map(m => ({ + label: m.name, + value: m.id, + hint: m.hint, + })) + const choice = guard(await p.select({ - message: 'Embedding model for indexing and querying skilld search', - options: EMBED_MODELS.map(m => ({ - label: m.label, - value: m.id, - hint: `${m.dimensions}d · ${m.hint}`, - })), + message: 'Embedding model: indexes and queries docs for skilld search', + options: [...builtIn, ...ollama], initialValue: current, })) @@ -276,14 +286,19 @@ async function configureEmbedModel(): Promise { updateConfig({ embedModel: choice === DEFAULT_EMBED_MODEL ? undefined : choice as string }) p.log.success(`Embedding model set to ${choice}`) - p.log.warn('Embedding model changed. Rebuild existing search indexes: skilld update --force') + p.log.warn('Run `skilld update` to rebuild existing search indexes with this model.') } -// Embedding device selection +// ── Embedding device selection ─────────────────────────────────────── async function configureEmbedDevice(): Promise { const config = readConfig() const current = config.embedDevice || DEFAULT_EMBED_DEVICE + + const isOllama = isOllamaEmbedModel(resolveEmbedModel(config.embedModel)) + if (isOllama) { + p.log.warn('The active embedding model runs inside Ollama, which manages its own device. This setting will have no effect until you switch to a built-in model.') + } const envOverride = process.env.SKILLD_EMBED_DEVICE?.trim() if (envOverride) @@ -297,7 +312,7 @@ async function configureEmbedDevice(): Promise { ) const choice = guard(await p.select({ - message: 'Embedding device where the model runs', + message: 'Embedding device: where the model runs', options: EMBED_DEVICES.map(d => ({ label: d.label, value: d.id, hint: d.hint })), initialValue: current, })) @@ -309,10 +324,13 @@ async function configureEmbedDevice(): Promise { updateConfig({ embedDevice: choice === DEFAULT_EMBED_DEVICE ? undefined : choice as string }) p.log.success(`Embedding device set to ${choice}`) - p.log.warn('Embedding device changed. Rebuild existing search indexes: skilld update --force') - if (choice !== DEFAULT_EMBED_DEVICE && choice !== 'cpu') - p.log.info('If indexing fails to start, switch back to Auto. The backend may be unavailable on this machine.') + if (!isOllama) + p.log.warn('Run `skilld update` to rebuild existing search indexes on this device.') + + if (choice !== DEFAULT_EMBED_DEVICE && choice !== 'cpu') { + p.log.info('If indexing fails to start, the backend is unavailable on this machine. Switch back to Auto.') + } } export const configCommandDef = defineCommand({ diff --git a/src/core/ollama-host.ts b/src/core/ollama-host.ts new file mode 100644 index 00000000..af0b8f19 --- /dev/null +++ b/src/core/ollama-host.ts @@ -0,0 +1,14 @@ +const HAS_SCHEME_RE = /^https?:\/\// +const TRAILING_SLASH_RE = /\/$/ + +/** + * Base URL for the local Ollama daemon, normalised so callers can append paths. + * + * Lives in `core/` rather than `agent/clis/ollama.ts` so the search worker can + * reach it without pulling in the agent registry. + */ +export function ollamaHost(): string { + const raw = process.env.OLLAMA_HOST || 'http://localhost:11434' + const withScheme = HAS_SCHEME_RE.test(raw) ? raw : `http://${raw}` + return withScheme.replace(TRAILING_SLASH_RE, '') +} diff --git a/src/core/semver.ts b/src/core/semver.ts index 063a41e4..531db780 100644 --- a/src/core/semver.ts +++ b/src/core/semver.ts @@ -75,4 +75,3 @@ export function pickLatestTag(distTags?: Record): PickedTa return pool[0] ?? null } - diff --git a/src/retriv/embedding-cache.ts b/src/retriv/embedding-cache.ts index 0b7efc88..1631a2cf 100644 --- a/src/retriv/embedding-cache.ts +++ b/src/retriv/embedding-cache.ts @@ -1,5 +1,6 @@ import type { DatabaseSync } from 'node:sqlite' import type { Embedding } from 'retriv' +import { createHash } from 'node:crypto' import { rmSync } from 'node:fs' import { join } from 'pathe' import { CACHE_DIR } from '../cache/index.ts' @@ -20,7 +21,6 @@ async function openDb(): Promise { db.exec('PRAGMA journal_mode=WAL') db.exec('PRAGMA busy_timeout=5000') db.exec(`CREATE TABLE IF NOT EXISTS embeddings (text_hash TEXT PRIMARY KEY, embedding BLOB NOT NULL)`) - db.exec(`CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`) _db = db return db } @@ -32,20 +32,21 @@ function closeDb(): void { } } -function createSqliteStorage(db: DatabaseSync) { +function createSqliteStorage(db: DatabaseSync, getNamespace: () => string) { const getStmt = db.prepare('SELECT embedding FROM embeddings WHERE text_hash = ?') const setStmt = db.prepare('INSERT OR IGNORE INTO embeddings (text_hash, embedding) VALUES (?, ?)') + const key = (hash: string) => `${getNamespace()}:${hash}` return { get: (hash: string): Embedding | null => { - const row = getStmt.get(hash) as { embedding: Buffer } | undefined + const row = getStmt.get(key(hash)) as { embedding: Buffer } | undefined if (!row) return null return new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.embedding.byteLength / 4) }, set: (hash: string, embedding: Embedding): void => { const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding) - setStmt.run(hash, Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength)) + setStmt.run(key(hash), Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength)) }, } } @@ -53,43 +54,32 @@ function createSqliteStorage(db: DatabaseSync) { /** * Wrap an embedding provider with the on-disk vector cache. * - * `model` identifies which embedder produced the cached vectors. Entries are - * keyed by text hash alone, so vectors from a different model would be served - * for the same text. Two models of equal width (bge-large and - * qwen3-embedding:0.6b are both 1024d) would silently mix embedding spaces and - * destroy ranking. Dimensions alone cannot catch that; the model id can. + * `identity` identifies the model and execution backend. It namespaces every + * text hash, so concurrent providers cannot read or overwrite each other's + * vectors. Dimensions join the namespace after the provider resolves. */ -export async function cachedEmbeddings(config: EmbeddingConfig, model?: string): Promise { +export async function cachedEmbeddings(config: EmbeddingConfig, identity: string): Promise { const { cachedEmbeddings: retrivCached } = await import('retriv/embeddings/cached') const db = await openDb() - const storage = createSqliteStorage(db) + let namespace: string | undefined + const storage = createSqliteStorage(db, () => { + if (!namespace) + throw new Error('Embedding cache used before its provider resolved') + return namespace + }) const originalResolve = config.resolve - const validatedConfig: EmbeddingConfig = { + const namespacedConfig: EmbeddingConfig = { async resolve() { const resolved = await originalResolve() - const getMetaStmt = db.prepare('SELECT value FROM meta WHERE key = ?') - const setMetaStmt = db.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)') - - const storedDims = getMetaStmt.get('dimensions') as { value: string } | undefined - const storedModel = getMetaStmt.get('model') as { value: string } | undefined - const dimsChanged = storedDims && Number(storedDims.value) !== resolved.dimensions - // A cache written before this key existed has unknown provenance, so - // treat a missing stored model as a mismatch once a model is supplied. - const modelChanged = model !== undefined && storedModel?.value !== model - - if (dimsChanged || modelChanged) - db.exec('DELETE FROM embeddings') - - setMetaStmt.run('dimensions', String(resolved.dimensions)) - if (model !== undefined) - setMetaStmt.run('model', model) - + namespace = createHash('sha256') + .update(`${identity}\0${resolved.dimensions}`) + .digest('hex') return resolved }, } - return retrivCached(validatedConfig, { storage }) + return retrivCached(namespacedConfig, { storage }) } export function clearEmbeddingCache(): void { diff --git a/src/retriv/index-embedding-identity.ts b/src/retriv/index-embedding-identity.ts deleted file mode 100644 index dbea2f43..00000000 --- a/src/retriv/index-embedding-identity.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import { existsSync } from 'node:fs' -import { DEFAULT_EMBEDDING_IDENTITY } from './models.ts' - -const META_TABLE = 'skilld_meta' -const IDENTITY_KEY = 'embedding_identity' - -export type IndexEmbeddingIdentityState - = | { _tag: 'Current' } - | { _tag: 'Missing' } - | { _tag: 'Mismatch', current: string, stored: string } - -function tableExists(db: DatabaseSync, name: string): boolean { - return db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name) !== undefined -} - -function hasIndexedDocuments(db: DatabaseSync): boolean { - if (!tableExists(db, 'documents_meta')) - return false - const row = db.prepare('SELECT EXISTS(SELECT 1 FROM documents_meta) AS found').get() as { found: number } - return row.found === 1 -} - -export function checkIndexEmbeddingIdentity(dbPath: string, current: string): IndexEmbeddingIdentityState { - if (dbPath === ':memory:' || !existsSync(dbPath)) - return { _tag: 'Missing' } - - const db = new DatabaseSync(dbPath, { open: true, readOnly: true }) - try { - const row = tableExists(db, META_TABLE) - ? db.prepare(`SELECT value FROM ${META_TABLE} WHERE key = ?`).get(IDENTITY_KEY) as { value: string } | undefined - : undefined - const stored = row?.value ?? (hasIndexedDocuments(db) ? DEFAULT_EMBEDDING_IDENTITY : undefined) - - if (!stored) - return { _tag: 'Missing' } - if (stored !== current) - return { _tag: 'Mismatch', current, stored } - return { _tag: 'Current' } - } - finally { - db.close() - } -} - -export function recordIndexEmbeddingIdentity(dbPath: string, identity: string): void { - if (dbPath === ':memory:') - return - const db = new DatabaseSync(dbPath) - try { - db.exec(`CREATE TABLE IF NOT EXISTS ${META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL)`) - db.prepare(`INSERT OR REPLACE INTO ${META_TABLE} (key, value) VALUES (?, ?)`).run(IDENTITY_KEY, identity) - } - finally { - db.close() - } -} diff --git a/src/retriv/index-identity.ts b/src/retriv/index-identity.ts new file mode 100644 index 00000000..88a45bf7 --- /dev/null +++ b/src/retriv/index-identity.ts @@ -0,0 +1,71 @@ +import type { SkilldConfig } from '../core/config.ts' +import { createHash } from 'node:crypto' +import { existsSync, rmSync } from 'node:fs' +import { resolveModelForPreset } from 'retriv/embeddings/model-info' +import { readConfig } from '../core/config.ts' +import { ollamaHost } from '../core/ollama-host.ts' +import { resolveEmbedDevice, resolveEmbedModel } from './models.ts' +import { isOllamaEmbedModel } from './ollama-embeddings.ts' + +const META_TABLE = 'skilld_meta' +const IDENTITY_KEY = 'embedding_identity' +const IDENTITY_VERSION = 'v1' + +export function resolveEmbeddingIdentity( + config: Pick = readConfig(), +): string { + const model = resolveEmbedModel(config.embedModel) + if (!isOllamaEmbedModel(model)) { + const resolvedModel = resolveModelForPreset(model, 'transformers.js') + return `${IDENTITY_VERSION}:${resolvedModel}@${resolveEmbedDevice(config.embedDevice) ?? 'auto'}` + } + + const host = createHash('sha256').update(ollamaHost()).digest('hex').slice(0, 16) + return `${IDENTITY_VERSION}:${model}@host:${host}` +} + +export function readIndexEmbeddingIdentity(dbPath: string): string | undefined { + if (!existsSync(dbPath)) + return undefined + const nodeSqlite = globalThis.process?.getBuiltinModule?.('node:sqlite') as typeof import('node:sqlite') | undefined + if (!nodeSqlite) + return undefined + + const db = new nodeSqlite.DatabaseSync(dbPath, { open: true, readOnly: true }) + try { + const table = db.prepare('SELECT 1 FROM sqlite_schema WHERE type = \'table\' AND name = ?').get(META_TABLE) + if (!table) + return undefined + const row = db.prepare(`SELECT value FROM ${META_TABLE} WHERE key = ?`).get(IDENTITY_KEY) as { value: string } | undefined + return row?.value + } + finally { + db.close() + } +} + +export function hasIndexEmbeddingIdentity(dbPath: string, identity: string): boolean { + return readIndexEmbeddingIdentity(dbPath) === identity +} + +export function removeStaleIndex(dbPath: string, identity: string): boolean { + if (!existsSync(dbPath) || hasIndexEmbeddingIdentity(dbPath, identity)) + return false + for (const path of [dbPath, `${dbPath}-shm`, `${dbPath}-wal`]) + rmSync(path, { force: true }) + return true +} + +export function writeIndexEmbeddingIdentity(dbPath: string, identity: string): void { + const nodeSqlite = globalThis.process?.getBuiltinModule?.('node:sqlite') as typeof import('node:sqlite') | undefined + if (!nodeSqlite) + throw new Error('SQLite is unavailable, so the search index identity cannot be saved') + const db = new nodeSqlite.DatabaseSync(dbPath) + try { + db.exec(`CREATE TABLE IF NOT EXISTS ${META_TABLE} (key TEXT PRIMARY KEY, value TEXT NOT NULL)`) + db.prepare(`INSERT OR REPLACE INTO ${META_TABLE} (key, value) VALUES (?, ?)`).run(IDENTITY_KEY, identity) + } + finally { + db.close() + } +} diff --git a/src/retriv/index-pipeline.ts b/src/retriv/index-pipeline.ts index 6d19cdcc..dc84778f 100644 --- a/src/retriv/index-pipeline.ts +++ b/src/retriv/index-pipeline.ts @@ -14,6 +14,7 @@ import { getPackageDbPath } from '../cache/index.ts' import { defaultFeatures, readConfig } from '../core/config.ts' import { resolvePkgDir } from '../core/prepare.ts' import { resolveEntryFiles } from '../sources/index.ts' +import { hasIndexEmbeddingIdentity, resolveEmbeddingIdentity } from './index-identity.ts' import { createIndex, listIndexIds, SearchDepsUnavailableError } from './index.ts' /** Max docs sent to the embedding pipeline to prevent oversized indexes */ @@ -62,7 +63,11 @@ export async function indexResources(opts: IndexResourcesOptions): Promise return const dbPath = getPackageDbPath(packageName, version) - const dbExists = existsSync(dbPath) + const storedDbExists = existsSync(dbPath) + const identity = resolveEmbeddingIdentity() + const dbExists = storedDbExists && hasIndexEmbeddingIdentity(dbPath, identity) + if (storedDbExists && !dbExists) + onProgress('Embedding settings changed, rebuilding search index') const allDocs = [...opts.docsToIndex] diff --git a/src/retriv/index.ts b/src/retriv/index.ts index 1e12c931..6510b692 100644 --- a/src/retriv/index.ts +++ b/src/retriv/index.ts @@ -1,8 +1,10 @@ import type { ChunkEntity, Document, IndexConfig, IndexPhase, IndexProgress, SearchFilter, SearchOptions, SearchResult, SearchSnippet } from './types.ts' +import { existsSync } from 'node:fs' import { readConfig } from '../core/config.ts' import { stripFrontmatter } from '../core/markdown.ts' -import { checkIndexEmbeddingIdentity, recordIndexEmbeddingIdentity } from './index-embedding-identity.ts' -import { getEmbeddingIdentity, resolveEmbedDevice, resolveEmbedModel } from './models.ts' +import { hasIndexEmbeddingIdentity, removeStaleIndex, resolveEmbeddingIdentity } from './index-identity.ts' +import { resolveEmbedDevice, resolveEmbedModel } from './models.ts' +import { isOllamaEmbedModel, ollamaEmbeddings } from './ollama-embeddings.ts' export type { ChunkEntity, Document, IndexConfig, IndexPhase, IndexProgress, SearchFilter, SearchOptions, SearchResult, SearchSnippet } @@ -16,14 +18,6 @@ export class SearchDepsUnavailableError extends Error { } } -export class EmbeddingIndexMismatchError extends Error { - constructor(dbPath: string, stored: string, current: string) { - super(`Search index uses ${stored}, but embedding settings resolve to ${current}. Rebuild indexes with: skilld update --force`) - this.name = 'EmbeddingIndexMismatchError' - this.cause = { dbPath, stored, current } - } -} - let _fts5Available: boolean | null = null /** @@ -54,18 +48,10 @@ function checkFts5(): boolean { } // Dynamic imports: retriv/chunkers/auto eagerly loads typescript which may not be installed (e.g. npx) -export async function getDb(config: Pick) { +async function openDb(config: Pick, identity: string) { if (!checkFts5()) throw new SearchDepsUnavailableError(new Error('FTS5 module not available'), 'SQLite FTS5 module not available. Search indexing skipped. On Windows, run from WSL where FTS5 is included.') - const userConfig = readConfig() - const embedModel = resolveEmbedModel(userConfig.embedModel) - const device = resolveEmbedDevice(userConfig.embedDevice) - const embeddingIdentity = getEmbeddingIdentity(embedModel, device) - const identityState = checkIndexEmbeddingIdentity(config.dbPath, embeddingIdentity) - if (identityState._tag === 'Mismatch') - throw new EmbeddingIndexMismatchError(config.dbPath, identityState.stored, identityState.current) - let createRetriv, autoChunker, sqliteMod, sqliteVec, transformersJs, cachedEmbeddings try { ;([ @@ -89,14 +75,24 @@ export async function getDb(config: Pick) { throw new SearchDepsUnavailableError(err) throw err } + const userConfig = readConfig() + const embedModel = resolveEmbedModel(userConfig.embedModel) + const device = resolveEmbedDevice(userConfig.embedDevice) + // Cache identity pairs model with device: cached vectors are only valid for + // the embedder that produced them, and backends can differ numerically. + // Ollama runs the model in its own process, so `device` does not apply there. + const isOllama = isOllamaEmbedModel(embedModel) const embeddings = await cachedEmbeddings( - transformersJs({ - model: embedModel, - ...(device ? { device } : {}), - }), - embeddingIdentity, + isOllama + ? ollamaEmbeddings(embedModel) + : transformersJs({ + model: embedModel, + // Omitted when `auto` so transformers.js keeps its own device resolution. + ...(device ? { device } : {}), + }), + identity, ) - const db = await createRetriv({ + return createRetriv({ driver: sqliteMod.default({ path: config.dbPath, embeddings, @@ -104,14 +100,24 @@ export async function getDb(config: Pick) { }), chunking: autoChunker(), }) - try { - recordIndexEmbeddingIdentity(config.dbPath, embeddingIdentity) - } - catch (error) { - await db.close?.() - throw error +} + +export async function getDb(config: Pick) { + const identity = resolveEmbeddingIdentity() + if (existsSync(config.dbPath) && !hasIndexEmbeddingIdentity(config.dbPath, identity)) { + throw new Error( + 'Search index uses different embedding settings. Run `skilld update` to rebuild it.', + ) } - return db + return openDb(config, identity) +} + +export async function getIndexDb( + config: Pick, + identity: string = resolveEmbeddingIdentity(), +) { + removeStaleIndex(config.dbPath, identity) + return openDb(config, identity) } /** diff --git a/src/retriv/models.ts b/src/retriv/models.ts index b667c4a5..db0cab37 100644 --- a/src/retriv/models.ts +++ b/src/retriv/models.ts @@ -1,5 +1,3 @@ -import { resolveModelForPreset } from 'retriv/embeddings/model-info' - /** * Local embedding models available to the search index. * @@ -9,13 +7,13 @@ import { resolveModelForPreset } from 'retriv/embeddings/model-info' * * Dimensions are fixed per model and sqlite-vec columns are fixed-width, so * switching a model or device invalidates existing indexes. Rebuild with - * `skilld update --force`. + * `skilld update`. */ export interface EmbedModelInfo { /** Model id passed to retriv (resolved to a Hugging Face repo internally) */ id: string label: string - /** Vector width determines index layout */ + /** Vector width, which determines index layout */ dimensions: number hint: string } @@ -76,21 +74,23 @@ export function resolveEmbedModel(configured?: string): string { * Everything else is opt-in because the fastest backend is hardware-specific: * on an Apple M5 Max `webgpu` measured 2.6 to 2.9 times faster than CPU. * `coreml` measured 3 to 8 times slower because it falls back to CPU for - * unsupported ops and pays for graph partitioning. + * unsupported operations and pays for graph partitioning. */ export interface EmbedDeviceInfo { - id: string + id: EmbedDeviceSetting label: string hint: string } export const DEFAULT_EMBED_DEVICE = 'auto' +export type EmbedDevice = 'cpu' | 'webgpu' | 'coreml' +export type EmbedDeviceSetting = typeof DEFAULT_EMBED_DEVICE | EmbedDevice -export const EMBED_DEVICES = [ +export const EMBED_DEVICES: readonly EmbedDeviceInfo[] = [ { id: 'auto', label: 'Auto', - hint: 'let transformers.js choose, CPU under Node', + hint: 'let transformers.js choose; CPU under Node', }, { id: 'cpu', @@ -100,17 +100,14 @@ export const EMBED_DEVICES = [ { id: 'webgpu', label: 'GPU (WebGPU)', - hint: 'fastest on Apple Silicon in testing, verify on your hardware', + hint: 'fastest on Apple Silicon in testing; verify on your hardware', }, { id: 'coreml', label: 'CoreML', - hint: 'Apple Neural Engine, measured slower than CPU for these models', + hint: 'Apple Neural Engine; measured slower than CPU for these models', }, -] as const satisfies readonly EmbedDeviceInfo[] - -export type EmbedDevice = typeof EMBED_DEVICES[number]['id'] -export type RuntimeEmbedDevice = Exclude +] export function getEmbedDeviceInfo(id: string): EmbedDeviceInfo | undefined { return EMBED_DEVICES.find(d => d.id === id) @@ -120,18 +117,11 @@ export function getEmbedDeviceInfo(id: string): EmbedDeviceInfo | undefined { * Resolve the execution device. Returns `undefined` for `auto` so the option * is omitted entirely and transformers.js keeps its own default resolution. */ -export function resolveEmbedDevice(configured?: string): RuntimeEmbedDevice | undefined { +export function resolveEmbedDevice(configured?: string): EmbedDevice | undefined { const fromEnv = process.env.SKILLD_EMBED_DEVICE?.trim() const value = fromEnv || configured || DEFAULT_EMBED_DEVICE - if (value === DEFAULT_EMBED_DEVICE) - return undefined - if (!getEmbedDeviceInfo(value)) - throw new Error(`Unsupported embedding device: ${value}`) - return value as RuntimeEmbedDevice + const device = getEmbedDeviceInfo(value) + if (!device) + throw new Error(`Unknown embedding device "${value}". Run \`skilld config\` to choose a supported device.`) + return device.id === DEFAULT_EMBED_DEVICE ? undefined : device.id } - -export function getEmbeddingIdentity(model: string, device?: string): string { - return `${resolveModelForPreset(model, 'transformers.js')}@${device ?? DEFAULT_EMBED_DEVICE}` -} - -export const DEFAULT_EMBEDDING_IDENTITY = getEmbeddingIdentity(DEFAULT_EMBED_MODEL) diff --git a/src/retriv/ollama-embeddings.ts b/src/retriv/ollama-embeddings.ts new file mode 100644 index 00000000..b0ac6f38 --- /dev/null +++ b/src/retriv/ollama-embeddings.ts @@ -0,0 +1,218 @@ +/** + * Ollama-backed embeddings for the search index. + * + * Talks to `/api/embed` directly rather than going through retriv's own Ollama + * provider, which requires the `ai` SDK and `ollama-ai-provider-v2`. skilld + * already speaks to Ollama over plain `fetch` for completions, so this keeps + * the dependency footprint unchanged. + */ +import type { Embedding } from 'retriv' +import { ollamaHost } from '../core/ollama-host.ts' + +/** Models are addressed as `ollama:`, matching the enhancement-model syntax. */ +export const OLLAMA_PREFIX = 'ollama:' + +/** Documents sent per `/api/embed` call. Keeps payloads and timeouts bounded. */ +const BATCH_SIZE = 64 + +/** Embedding runs are slow on large models; discovery stays snappy separately. */ +const EMBED_TIMEOUT_MS = 120_000 +const DISCOVERY_TIMEOUT_MS = 1500 + +export function isOllamaEmbedModel(id: string): boolean { + return id.startsWith(OLLAMA_PREFIX) +} + +export function stripOllamaPrefix(id: string): string { + return id.startsWith(OLLAMA_PREFIX) ? id.slice(OLLAMA_PREFIX.length) : id +} + +interface OllamaShowResponse { + capabilities?: string[] + model_info?: Record +} + +interface OllamaTagsResponse { + models?: Array<{ + name: string + size?: number + details?: { parameter_size?: string, quantization_level?: string } + }> +} + +/** + * Pull a value out of `model_info`, whose keys are architecture-prefixed + * (`qwen3.embedding_length`, `gemma3.context_length`, …). + */ +function readModelInfo(info: Record | undefined, suffix: string): number | undefined { + if (!info) + return undefined + for (const [key, value] of Object.entries(info)) { + if (key.endsWith(suffix) && typeof value === 'number') + return value + } + return undefined +} + +async function showModel(name: string, timeout: number): Promise { + const res = await fetch(`${ollamaHost()}/api/show`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model: name }), + signal: AbortSignal.timeout(timeout), + }).catch(() => null) + if (!res?.ok) + return null + return await res.json().catch(() => null) as OllamaShowResponse | null +} + +export interface OllamaEmbedModelInfo { + /** Prefixed id, e.g. `ollama:qwen3-embedding:0.6b` */ + id: string + name: string + dimensions?: number + hint: string +} + +/** + * Locally-pulled Ollama models that advertise the `embedding` capability. + * + * Returns `[]` when the daemon is unreachable. Discovery must never block or + * throw, it just contributes nothing to the picker. + */ +export async function getAvailableOllamaEmbedModels(): Promise { + const res = await fetch(`${ollamaHost()}/api/tags`, { signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS) }) + .catch(() => null) + if (!res?.ok) + return [] + + const data = await res.json().catch(() => null) as OllamaTagsResponse | null + if (!data?.models?.length) + return [] + + const checked = await Promise.all(data.models.map(async (m): Promise => { + const info = await showModel(m.name, DISCOVERY_TIMEOUT_MS) + // Unlike completions we cannot fail open: a chat model returns an error + // from /api/embed, so an unconfirmed model would break indexing later. + if (!info?.capabilities?.includes('embedding')) + return null + + const dimensions = readModelInfo(info.model_info, '.embedding_length') + const params = m.details?.parameter_size + const detail = [dimensions ? `${dimensions}d` : null, params].filter(Boolean).join(' · ') + return { + id: `${OLLAMA_PREFIX}${m.name}`, + name: m.name, + dimensions, + hint: detail ? `local · ${detail}` : 'local', + } + })) + + return checked.filter((m): m is OllamaEmbedModelInfo => m !== null) +} + +function l2Normalize(vector: unknown, expectedDimensions?: number): Float32Array { + if (!Array.isArray(vector) || !vector.every(value => typeof value === 'number' && Number.isFinite(value))) + throw new Error('Ollama returned an invalid embedding vector') + if (expectedDimensions !== undefined && vector.length !== expectedDimensions) { + throw new Error( + `Ollama returned ${vector.length} dimensions, expected ${expectedDimensions}`, + ) + } + let sum = 0 + for (const value of vector) + sum += value * value + const norm = Math.sqrt(sum) + const out = new Float32Array(vector.length) + if (norm === 0) + throw new Error('Ollama returned a zero vector') + if (!Number.isFinite(norm)) + throw new Error('Ollama returned an invalid vector magnitude') + for (let i = 0; i < vector.length; i++) + out[i] = vector[i]! / norm + return out +} + +async function embedBatch(model: string, input: string[], expectedDimensions?: number): Promise { + const res = await fetch(`${ollamaHost()}/api/embed`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model, input }), + signal: AbortSignal.timeout(EMBED_TIMEOUT_MS), + }).catch((err) => { + throw new Error(`Could not reach Ollama at ${ollamaHost()}: ${err instanceof Error ? err.message : String(err)}`) + }) + + const data = await res.json().catch(() => null) as { embeddings?: unknown[], error?: string } | null + if (!res.ok || data?.error) { + const message = data?.error || `HTTP ${res.status}` + throw new Error(`Ollama embedding failed for ${model}: ${message}`) + } + if (!Array.isArray(data?.embeddings) || data.embeddings.length === 0) + throw new Error(`Ollama returned no embeddings for ${model}`) + if (data.embeddings.length !== input.length) { + throw new Error( + `Ollama returned ${data.embeddings.length} embeddings for ${input.length} inputs`, + ) + } + + return data.embeddings.map(vector => l2Normalize(vector, expectedDimensions)) +} + +/** + * Build a retriv `EmbeddingConfig` backed by Ollama. + * + * `id` may be prefixed or bare. Dimensions come from `/api/show` when the model + * reports them, otherwise from a probe embedding. + */ +export function ollamaEmbeddings(id: string): { + resolve: () => Promise<{ embedder: (texts: string[]) => Promise, dimensions: number, maxTokens?: number }> +} { + const model = stripOllamaPrefix(id) + let cached: { embedder: (texts: string[]) => Promise, dimensions: number, maxTokens?: number } | null = null + + return { + async resolve() { + if (cached) + return cached + + const info = await showModel(model, DISCOVERY_TIMEOUT_MS) + if (!info) { + throw new Error( + `Ollama is not reachable at ${ollamaHost()}. Start it with \`ollama serve\`, ` + + `or pick a built-in model with \`skilld config\`.`, + ) + } + if (!info.capabilities?.includes('embedding')) { + throw new Error( + `Ollama model "${model}" does not support embeddings. ` + + `Pull an embedding model, for example \`ollama pull qwen3-embedding\`.`, + ) + } + + let dimensions = readModelInfo(info.model_info, '.embedding_length') + if (!dimensions) { + const [probe] = await embedBatch(model, ['dimension probe']) + dimensions = probe?.length + } + if (!dimensions) + throw new Error(`Could not determine embedding dimensions for Ollama model "${model}"`) + + const maxTokens = readModelInfo(info.model_info, '.context_length') + + const embedder = async (texts: string[]): Promise => { + if (texts.length === 0) + return [] + const out: Embedding[] = [] + for (let i = 0; i < texts.length; i += BATCH_SIZE) { + const batch = await embedBatch(model, texts.slice(i, i + BATCH_SIZE), dimensions) + out.push(...batch) + } + return out + } + + cached = { embedder, dimensions, maxTokens } + return cached + }, + } +} diff --git a/src/retriv/pool.ts b/src/retriv/pool.ts index b54036ba..de432799 100644 --- a/src/retriv/pool.ts +++ b/src/retriv/pool.ts @@ -4,16 +4,11 @@ import { existsSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { Worker } from 'node:worker_threads' import { dirname, join } from 'pathe' -import { EmbeddingIndexMismatchError, SearchDepsUnavailableError } from './index.ts' +import { SearchDepsUnavailableError } from './index.ts' function reconstructError(message: string, name?: string): Error { if (name === 'SearchDepsUnavailableError') return new SearchDepsUnavailableError(undefined, message) - if (name === 'EmbeddingIndexMismatchError') { - const error = new Error(message) - error.name = EmbeddingIndexMismatchError.name - return error - } return new Error(message) } diff --git a/src/retriv/worker.ts b/src/retriv/worker.ts index fe480e25..8c3cfff3 100644 --- a/src/retriv/worker.ts +++ b/src/retriv/worker.ts @@ -55,12 +55,19 @@ if (parentPort) { }, } - const { getDb } = await import('./index.ts') - const db = await getDb(config) - if (msg.removeIds?.length) - await db.remove?.(msg.removeIds) - await db.index(documents, { onProgress: config.onProgress }) - await db.close?.() + const { getIndexDb } = await import('./index.ts') + const { resolveEmbeddingIdentity, writeIndexEmbeddingIdentity } = await import('./index-identity.ts') + const identity = resolveEmbeddingIdentity() + const db = await getIndexDb(config, identity) + try { + if (msg.removeIds?.length) + await db.remove?.(msg.removeIds) + await db.index(documents, { onProgress: config.onProgress }) + } + finally { + await db.close?.() + } + writeIndexEmbeddingIdentity(dbPath, identity) parentPort!.postMessage({ type: 'done', id } satisfies WorkerDoneResponse) } diff --git a/test/unit/embed-models.test.ts b/test/unit/embed-models.test.ts index f3c698c3..3b5439cf 100644 --- a/test/unit/embed-models.test.ts +++ b/test/unit/embed-models.test.ts @@ -1,6 +1,6 @@ import { getModelDimensions, resolveModelForPreset } from 'retriv/embeddings/model-info' import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { DEFAULT_EMBED_DEVICE, DEFAULT_EMBED_MODEL, EMBED_DEVICES, EMBED_MODELS, getEmbedDeviceInfo, getEmbedModelInfo, getEmbeddingIdentity, resolveEmbedDevice, resolveEmbedModel } from '../../src/retriv/models.ts' +import { DEFAULT_EMBED_DEVICE, DEFAULT_EMBED_MODEL, EMBED_DEVICES, EMBED_MODELS, getEmbedDeviceInfo, getEmbedModelInfo, resolveEmbedDevice, resolveEmbedModel } from '../../src/retriv/models.ts' describe('resolveEmbedModel', () => { let original: string | undefined @@ -116,15 +116,13 @@ describe('resolveEmbedDevice', () => { expect(resolveEmbedDevice('webgpu')).toBeUndefined() }) - it('rejects an unsupported device', () => { - expect(() => resolveEmbedDevice('invalid')).toThrow('Unsupported embedding device: invalid') + it('rejects an unknown configured device', () => { + expect(() => resolveEmbedDevice('quantum')).toThrow(/unknown embedding device/i) }) -}) -describe('getEmbeddingIdentity', () => { - it('uses the resolved model repo and device', () => { - expect(getEmbeddingIdentity('bge-small-en-v1.5', 'webgpu')) - .toBe('Xenova/bge-small-en-v1.5@webgpu') + it('rejects an unknown device from the environment', () => { + process.env.SKILLD_EMBED_DEVICE = 'quantum' + expect(() => resolveEmbedDevice('cpu')).toThrow(/unknown embedding device/i) }) }) diff --git a/test/unit/embedding-cache.test.ts b/test/unit/embedding-cache.test.ts index 99215ecd..12a6332d 100644 --- a/test/unit/embedding-cache.test.ts +++ b/test/unit/embedding-cache.test.ts @@ -43,7 +43,7 @@ describe('embedding-cache', () => { it('computes embeddings on first call (cache miss)', async () => { const { config, calls } = fakeEmbeddingConfig() - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() const result = await embedder(['hello', 'world']) @@ -55,7 +55,7 @@ describe('embedding-cache', () => { it('serves cached embeddings on second call (cache hit)', async () => { const { config, calls } = fakeEmbeddingConfig() - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() await embedder(['hello', 'world']) @@ -68,7 +68,7 @@ describe('embedding-cache', () => { it('computes only missed texts on partial cache hit', async () => { const { config, calls } = fakeEmbeddingConfig() - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() await embedder(['hello']) @@ -87,7 +87,7 @@ describe('embedding-cache', () => { return new Float32Array([counter, counter * 10]) }) }) - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() await embedder(['a', 'b']) @@ -101,17 +101,17 @@ describe('embedding-cache', () => { expect([...result[2] as Float32Array]).toEqual([1, 10]) }) - it('wipes cache on dimension mismatch', async () => { + it('does not serve cached vectors from another dimension', async () => { // First: populate with 4-dim embeddings const { config: config4, calls: calls4 } = fakeEmbeddingConfig(4) - const wrapped4 = await cachedEmbeddings(config4) + const wrapped4 = await cachedEmbeddings(config4, 'model-a@cpu') const { embedder: embedder4 } = await wrapped4.resolve() await embedder4(['hello']) expect(calls4).toHaveLength(1) // Second: resolve with 8-dim → should wipe, recompute const { config: config8, calls: calls8 } = fakeEmbeddingConfig(8) - const wrapped8 = await cachedEmbeddings(config8) + const wrapped8 = await cachedEmbeddings(config8, 'model-a@cpu') const { embedder: embedder8 } = await wrapped8.resolve() const result = await embedder8(['hello']) @@ -120,59 +120,20 @@ describe('embedding-cache', () => { expect((result[0] as Float32Array).length).toBe(8) }) - it('wipes cache when the model changes at the same dimension', async () => { - const { config: first } = fakeEmbeddingConfig(4, async texts => - texts.map(() => new Float32Array(4).fill(1))) - const firstWrapped = await cachedEmbeddings(first, 'model-a@cpu') - const { embedder: firstEmbedder } = await firstWrapped.resolve() - await firstEmbedder(['hello']) - - const { config: second, calls } = fakeEmbeddingConfig(4, async (texts) => { - calls.push(texts) - return texts.map(() => new Float32Array(4).fill(2)) - }) - const secondWrapped = await cachedEmbeddings(second, 'model-b@cpu') - const { embedder: secondEmbedder } = await secondWrapped.resolve() - - const result = await secondEmbedder(['hello']) - expect(calls).toEqual([['hello']]) - expect([...result[0] as Float32Array]).toEqual([2, 2, 2, 2]) - }) - - it('keeps cache when model and device are unchanged', async () => { - const { config: first } = fakeEmbeddingConfig(4, async texts => - texts.map(() => new Float32Array(4).fill(1))) - const firstWrapped = await cachedEmbeddings(first, 'model-a@cpu') - const { embedder: firstEmbedder } = await firstWrapped.resolve() - await firstEmbedder(['hello']) - - const { config: second, calls } = fakeEmbeddingConfig(4) - const secondWrapped = await cachedEmbeddings(second, 'model-a@cpu') - const { embedder: secondEmbedder } = await secondWrapped.resolve() - - const result = await secondEmbedder(['hello']) - expect(calls).toHaveLength(0) - expect([...result[0] as Float32Array]).toEqual([1, 1, 1, 1]) - }) - - it('wipes a cache with unknown legacy identity', async () => { - const { config: legacy } = fakeEmbeddingConfig(4, async texts => - texts.map(() => new Float32Array(4).fill(1))) - const legacyWrapped = await cachedEmbeddings(legacy) - const { embedder: legacyEmbedder } = await legacyWrapped.resolve() - await legacyEmbedder(['hello']) + it('does not serve cached vectors from another model', async () => { + const { config, calls } = fakeEmbeddingConfig() + const first = await cachedEmbeddings(config, 'model-a@cpu') + await (await first.resolve()).embedder(['hello']) - const { config: current, calls } = fakeEmbeddingConfig(4) - const currentWrapped = await cachedEmbeddings(current, 'model-a@cpu') - const { embedder: currentEmbedder } = await currentWrapped.resolve() + const second = await cachedEmbeddings(config, 'model-b@cpu') + await (await second.resolve()).embedder(['hello']) - await currentEmbedder(['hello']) - expect(calls).toEqual([['hello']]) + expect(calls).toEqual([['hello'], ['hello']]) }) it('clearEmbeddingCache removes the db file', async () => { const { config } = fakeEmbeddingConfig() - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() await embedder(['hello']) @@ -187,12 +148,12 @@ describe('embedding-cache', () => { const { config, calls } = fakeEmbeddingConfig() // First resolve + embed - const wrapped1 = await cachedEmbeddings(config) + const wrapped1 = await cachedEmbeddings(config, 'model-a@cpu') const { embedder: e1 } = await wrapped1.resolve() await e1(['hello']) // Second resolve (simulates new process opening same DB) - const wrapped2 = await cachedEmbeddings(config) + const wrapped2 = await cachedEmbeddings(config, 'model-a@cpu') const { embedder: e2 } = await wrapped2.resolve() await e2(['hello']) @@ -202,7 +163,7 @@ describe('embedding-cache', () => { it('handles empty input', async () => { const { config, calls } = fakeEmbeddingConfig() - const wrapped = await cachedEmbeddings(config) + const wrapped = await cachedEmbeddings(config, 'model-a@cpu') const { embedder } = await wrapped.resolve() const result = await embedder([]) diff --git a/test/unit/git-skills.test.ts b/test/unit/git-skills.test.ts index d031175a..99c9ba99 100644 --- a/test/unit/git-skills.test.ts +++ b/test/unit/git-skills.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'pathe' import { describe, expect, it } from 'vitest' @@ -209,19 +209,26 @@ author: someone }) it('discovers skills from agent-specific skill directories', async () => { - const fixture = join(__dirname, '../fixtures/mock-skills-repo-agent-dirs') - const { skills } = await fetchGitSkills({ type: 'local', localPath: fixture }) - - expect(skills.map(s => s.name).sort()).toEqual([ - 'agents-skill', - 'claude-skill', - 'github-skill', - ]) - expect(skills.map(s => s.path).sort()).toEqual([ - '.agents/skills/agents-skill', - '.claude/skills/claude-skill', - '.github/skills/github-skill', - ]) + const fixture = mkdtempSync(join(tmpdir(), 'skilld-agent-skill-dirs-')) + const expected = [ + { name: 'agents-skill', path: '.agents/skills/agents-skill' }, + { name: 'claude-skill', path: '.claude/skills/claude-skill' }, + { name: 'github-skill', path: '.github/skills/github-skill' }, + ] + for (const skill of expected) { + const skillDir = join(fixture, skill.path) + mkdirSync(skillDir, { recursive: true }) + writeFileSync(join(skillDir, 'SKILL.md'), `---\nname: ${skill.name}\n---\n`) + } + + try { + const { skills } = await fetchGitSkills({ type: 'local', localPath: fixture }) + expect(skills.map(s => s.name).sort()).toEqual(expected.map(s => s.name)) + expect(skills.map(s => s.path).sort()).toEqual(expected.map(s => s.path)) + } + finally { + rmSync(fixture, { recursive: true, force: true }) + } }) }) diff --git a/test/unit/index-embedding-identity.test.ts b/test/unit/index-embedding-identity.test.ts deleted file mode 100644 index 98e9a728..00000000 --- a/test/unit/index-embedding-identity.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { DatabaseSync } from 'node:sqlite' -import { mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { join } from 'pathe' -import { afterEach, describe, expect, it } from 'vitest' -import { EmbeddingIndexMismatchError, getDb } from '../../src/retriv/index.ts' -import { checkIndexEmbeddingIdentity, recordIndexEmbeddingIdentity } from '../../src/retriv/index-embedding-identity.ts' - -const dirs: string[] = [] - -function dbPath(): string { - const dir = mkdtempSync(join(tmpdir(), 'skilld-index-identity-')) - dirs.push(dir) - return join(dir, 'search.db') -} - -afterEach(() => { - for (const dir of dirs.splice(0)) - rmSync(dir, { recursive: true, force: true }) -}) - -describe('index embedding identity', () => { - it('rejects an equal-width model change', () => { - const path = dbPath() - recordIndexEmbeddingIdentity(path, 'Xenova/bge-large-en-v1.5@auto') - - expect(checkIndexEmbeddingIdentity(path, 'Xenova/bge-m3@auto')).toEqual({ - _tag: 'Mismatch', - current: 'Xenova/bge-m3@auto', - stored: 'Xenova/bge-large-en-v1.5@auto', - }) - }) - - it('blocks opening an index with a different model', async () => { - const path = dbPath() - recordIndexEmbeddingIdentity(path, 'Xenova/bge-large-en-v1.5@auto') - const originalModel = process.env.SKILLD_EMBED_MODEL - const originalDevice = process.env.SKILLD_EMBED_DEVICE - process.env.SKILLD_EMBED_MODEL = 'bge-m3' - process.env.SKILLD_EMBED_DEVICE = 'auto' - - try { - await expect(getDb({ dbPath: path })).rejects.toBeInstanceOf(EmbeddingIndexMismatchError) - } - finally { - if (originalModel === undefined) - delete process.env.SKILLD_EMBED_MODEL - else - process.env.SKILLD_EMBED_MODEL = originalModel - if (originalDevice === undefined) - delete process.env.SKILLD_EMBED_DEVICE - else - process.env.SKILLD_EMBED_DEVICE = originalDevice - } - }) - - it('treats existing pre-identity indexes as the old default', () => { - const path = dbPath() - const db = new DatabaseSync(path) - db.exec('CREATE TABLE documents_meta (id TEXT PRIMARY KEY)') - db.prepare('INSERT INTO documents_meta (id) VALUES (?)').run('doc') - db.close() - - expect(checkIndexEmbeddingIdentity(path, 'Xenova/bge-m3@auto')).toEqual({ - _tag: 'Mismatch', - current: 'Xenova/bge-m3@auto', - stored: 'Xenova/bge-small-en-v1.5@auto', - }) - }) - - it('accepts the identity recorded for a new index', () => { - const path = dbPath() - recordIndexEmbeddingIdentity(path, 'Xenova/bge-m3@webgpu') - - expect(checkIndexEmbeddingIdentity(path, 'Xenova/bge-m3@webgpu')).toEqual({ - _tag: 'Current', - }) - }) -}) diff --git a/test/unit/index-identity.test.ts b/test/unit/index-identity.test.ts new file mode 100644 index 00000000..5ab97945 --- /dev/null +++ b/test/unit/index-identity.test.ts @@ -0,0 +1,52 @@ +import { mkdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { DatabaseSync } from 'node:sqlite' +import { join } from 'pathe' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + hasIndexEmbeddingIdentity, + readIndexEmbeddingIdentity, + removeStaleIndex, + resolveEmbeddingIdentity, + writeIndexEmbeddingIdentity, +} from '../../src/retriv/index-identity.ts' + +const TEST_DIR = join(tmpdir(), 'skilld-test-index-identity') +const DB_PATH = join(TEST_DIR, 'search.db') + +afterEach(() => { + vi.unstubAllEnvs() + rmSync(TEST_DIR, { recursive: true, force: true }) +}) + +function createIndexFile(): void { + mkdirSync(TEST_DIR, { recursive: true }) + new DatabaseSync(DB_PATH).close() +} + +describe('search index embedding identity', () => { + it('resolves transformer presets before identifying the provider', () => { + vi.stubEnv('SKILLD_EMBED_MODEL', '') + vi.stubEnv('SKILLD_EMBED_DEVICE', '') + expect(resolveEmbeddingIdentity({ + embedModel: 'bge-small-en-v1.5', + embedDevice: 'webgpu', + })).toBe('v1:Xenova/bge-small-en-v1.5@webgpu') + }) + + it('persists the model identity in the search database', () => { + createIndexFile() + writeIndexEmbeddingIdentity(DB_PATH, 'model-a@cpu') + + expect(readIndexEmbeddingIdentity(DB_PATH)).toBe('model-a@cpu') + expect(hasIndexEmbeddingIdentity(DB_PATH, 'model-a@cpu')).toBe(true) + }) + + it('removes an index built with different embedding settings', () => { + createIndexFile() + writeIndexEmbeddingIdentity(DB_PATH, 'model-a@cpu') + + expect(removeStaleIndex(DB_PATH, 'model-b@cpu')).toBe(true) + expect(readIndexEmbeddingIdentity(DB_PATH)).toBeUndefined() + }) +}) diff --git a/test/unit/ollama-embeddings.test.ts b/test/unit/ollama-embeddings.test.ts new file mode 100644 index 00000000..c288df7a --- /dev/null +++ b/test/unit/ollama-embeddings.test.ts @@ -0,0 +1,239 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + getAvailableOllamaEmbedModels, + isOllamaEmbedModel, + ollamaEmbeddings, + stripOllamaPrefix, +} from '../../src/retriv/ollama-embeddings.ts' + +const SHOW_EMBEDDING = { + capabilities: ['embedding'], + model_info: { 'qwen3.embedding_length': 1024, 'qwen3.context_length': 32768 }, +} + +function jsonResponse(body: unknown, ok = true, status = 200) { + return { + ok, + status, + json: async () => body, + } as unknown as Response +} + +/** Route mocked fetch by URL path so tests read as request/response pairs. */ +function mockOllama(routes: Record Response | Promise>) { + return vi.fn(async (url: string | URL) => { + const path = new URL(String(url)).pathname + const handler = routes[path] + if (!handler) + throw new Error(`unexpected request to ${path}`) + return handler() + }) +} + +describe('ollama embed model ids', () => { + it('detects the prefix', () => { + expect(isOllamaEmbedModel('ollama:nomic-embed-text')).toBe(true) + expect(isOllamaEmbedModel('bge-small-en-v1.5')).toBe(false) + }) + + // Model names contain colons (`qwen3-embedding:0.6b`), so only the leading + // prefix may be stripped. + it('strips only the leading prefix', () => { + expect(stripOllamaPrefix('ollama:qwen3-embedding:0.6b')).toBe('qwen3-embedding:0.6b') + expect(stripOllamaPrefix('qwen3-embedding:0.6b')).toBe('qwen3-embedding:0.6b') + }) +}) + +describe('ollamaEmbeddings', () => { + const original = globalThis.fetch + + afterEach(() => { + globalThis.fetch = original + vi.restoreAllMocks() + }) + + it('reads dimensions and max tokens from /api/show without a probe embed', async () => { + const fetchMock = mockOllama({ '/api/show': () => jsonResponse(SHOW_EMBEDDING) }) + globalThis.fetch = fetchMock as unknown as typeof fetch + + const resolved = await ollamaEmbeddings('ollama:qwen3-embedding:0.6b').resolve() + + expect(resolved.dimensions).toBe(1024) + expect(resolved.maxTokens).toBe(32768) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('falls back to a probe embed when the model reports no embedding_length', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['embedding'], model_info: {} }), + '/api/embed': () => jsonResponse({ embeddings: [[0, 1, 0]] }), + }) as unknown as typeof fetch + + const resolved = await ollamaEmbeddings('ollama:mystery-model').resolve() + expect(resolved.dimensions).toBe(3) + }) + + it('normalizes returned vectors to unit length', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ + capabilities: ['embedding'], + model_info: { 'test.embedding_length': 2 }, + }), + // Deliberately unnormalized: the index scores by L2 distance and assumes + // unit vectors, so magnitude must not leak into ranking. + '/api/embed': () => jsonResponse({ embeddings: [[3, 4], [0, 10]] }), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + const [a, b] = await embedder(['one', 'two']) + + const norm = (v: ArrayLike) => Math.sqrt(Array.from(v).reduce((s, x) => s + x * x, 0)) + expect(norm(a!)).toBeCloseTo(1, 6) + expect(norm(b!)).toBeCloseTo(1, 6) + // [3, 4] has magnitude 5, so it normalizes to [0.6, 0.8]. + expect(a![0]).toBeCloseTo(0.6, 6) + expect(a![1]).toBeCloseTo(0.8, 6) + }) + + it('returns an empty array without calling the API for no input', async () => { + const fetchMock = mockOllama({ '/api/show': () => jsonResponse(SHOW_EMBEDDING) }) + globalThis.fetch = fetchMock as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + expect(await embedder([])).toEqual([]) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('explains how to start Ollama when the daemon is unreachable', async () => { + globalThis.fetch = vi.fn(async () => { + throw new Error('ECONNREFUSED') + }) as unknown as typeof fetch + + await expect(ollamaEmbeddings('ollama:x').resolve()).rejects.toThrow(/not reachable/i) + }) + + it('rejects a model that does not support embeddings', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['completion'], model_info: {} }), + }) as unknown as typeof fetch + + await expect(ollamaEmbeddings('ollama:llama3').resolve()).rejects.toThrow(/does not support embeddings/i) + }) + + it('rejects a model whose embedding capability is missing', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ model_info: { 'llama.embedding_length': 4096 } }), + }) as unknown as typeof fetch + + await expect(ollamaEmbeddings('ollama:llama3').resolve()).rejects.toThrow(/does not support embeddings/i) + }) + + it('rejects a response with fewer vectors than inputs', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['embedding'], model_info: { 'test.embedding_length': 2 } }), + '/api/embed': () => jsonResponse({ embeddings: [[1, 0]] }), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + await expect(embedder(['one', 'two'])).rejects.toThrow(/returned 1 embeddings for 2 inputs/i) + }) + + it('rejects a vector with the wrong dimensions', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['embedding'], model_info: { 'test.embedding_length': 3 } }), + '/api/embed': () => jsonResponse({ embeddings: [[1, 0]] }), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + await expect(embedder(['one'])).rejects.toThrow(/returned 2 dimensions, expected 3/i) + }) + + it('rejects a zero vector', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['embedding'], model_info: { 'test.embedding_length': 2 } }), + '/api/embed': () => jsonResponse({ embeddings: [[0, 0]] }), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + await expect(embedder(['one'])).rejects.toThrow(/zero vector/i) + }) + + it('rejects a vector whose magnitude overflows', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse({ capabilities: ['embedding'], model_info: { 'test.embedding_length': 2 } }), + '/api/embed': () => jsonResponse({ embeddings: [[Number.MAX_VALUE, Number.MAX_VALUE]] }), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + await expect(embedder(['one'])).rejects.toThrow(/invalid vector magnitude/i) + }) + + it('surfaces the Ollama error message when embedding fails', async () => { + globalThis.fetch = mockOllama({ + '/api/show': () => jsonResponse(SHOW_EMBEDDING), + '/api/embed': () => jsonResponse({ error: 'model "x" not found, try pulling it first' }, false, 404), + }) as unknown as typeof fetch + + const { embedder } = await ollamaEmbeddings('ollama:x').resolve() + await expect(embedder(['hi'])).rejects.toThrow(/try pulling it first/) + }) +}) + +describe('getAvailableOllamaEmbedModels', () => { + const original = globalThis.fetch + + beforeEach(() => { + delete process.env.OLLAMA_HOST + }) + + afterEach(() => { + globalThis.fetch = original + vi.restoreAllMocks() + }) + + // Discovery feeds a config menu; it must degrade to an empty list rather + // than block or throw when Ollama is not installed. + it('returns an empty list when the daemon is unreachable', async () => { + globalThis.fetch = vi.fn(async () => { + throw new Error('ECONNREFUSED') + }) as unknown as typeof fetch + + await expect(getAvailableOllamaEmbedModels()).resolves.toEqual([]) + }) + + it('keeps only models advertising the embedding capability', async () => { + globalThis.fetch = vi.fn(async (url: string | URL, init?: RequestInit) => { + const path = new URL(String(url)).pathname + if (path === '/api/tags') { + return jsonResponse({ + models: [ + { name: 'qwen3-embedding:0.6b', details: { parameter_size: '595M' } }, + { name: 'llama3:8b', details: { parameter_size: '8B' } }, + ], + }) + } + const body = JSON.parse(String(init?.body)) as { model: string } + return jsonResponse(body.model.startsWith('qwen3') + ? SHOW_EMBEDDING + : { capabilities: ['completion'], model_info: {} }) + }) as unknown as typeof fetch + + const models = await getAvailableOllamaEmbedModels() + + expect(models).toHaveLength(1) + expect(models[0]!.id).toBe('ollama:qwen3-embedding:0.6b') + expect(models[0]!.dimensions).toBe(1024) + expect(models[0]!.hint).toContain('1024d') + }) + + it('drops models whose capabilities cannot be confirmed', async () => { + globalThis.fetch = vi.fn(async (url: string | URL) => { + const path = new URL(String(url)).pathname + if (path === '/api/tags') + return jsonResponse({ models: [{ name: 'mystery' }] }) + return jsonResponse({}, false, 500) + }) as unknown as typeof fetch + + await expect(getAvailableOllamaEmbedModels()).resolves.toEqual([]) + }) +}) diff --git a/test/unit/sync-pipeline.test.ts b/test/unit/sync-pipeline.test.ts index 7191a8f8..ba1b7ce8 100644 --- a/test/unit/sync-pipeline.test.ts +++ b/test/unit/sync-pipeline.test.ts @@ -171,6 +171,11 @@ vi.mock('../../src/core/lockfile', () => ({ writeLock: vi.fn(), })) +vi.mock('../../src/retriv/index-identity', () => ({ + hasIndexEmbeddingIdentity: vi.fn(() => true), + resolveEmbeddingIdentity: vi.fn(() => 'v1:model-a@auto'), +})) + vi.mock('../../src/retriv', async (importOriginal) => { const orig = await importOriginal() return { ...orig, createIndex: vi.fn(), listIndexIds: vi.fn().mockResolvedValue([]) } @@ -188,6 +193,7 @@ const { fetchReleaseNotes, isGhAvailable, isShallowGitDocs, resolveEntryFiles } const { registerProject } = await import('../../src/core/config') const { writeLock } = await import('../../src/core/lockfile') const { createIndex, listIndexIds } = await import('../../src/retriv') +const { hasIndexEmbeddingIdentity, resolveEmbeddingIdentity } = await import('../../src/retriv/index-identity') const { getShippedSkills, linkShippedSkill, resolvePkgDir } = await import('../../src/core/prepare') const { @@ -216,6 +222,8 @@ describe('sync-shared', () => { vi.mocked(isShallowGitDocs).mockReturnValue(false) vi.mocked(resolveEntryFiles).mockResolvedValue([]) vi.mocked(fetchReleaseNotes).mockResolvedValue([]) + vi.mocked(hasIndexEmbeddingIdentity).mockReturnValue(true) + vi.mocked(resolveEmbeddingIdentity).mockReturnValue('v1:model-a@auto') }) // ── 1. classifyCachedDoc ── @@ -405,6 +413,21 @@ describe('sync-shared', () => { expect(onProgress).toHaveBeenCalledWith('Search index up to date') }) + it('rebuilds every document when embedding settings changed', async () => { + vi.mocked(existsSync).mockReturnValue(true) + vi.mocked(hasIndexEmbeddingIdentity).mockReturnValue(false) + vi.mocked(resolvePkgDir).mockReturnValue(null) + const docs = [ + { id: 'a.md', content: 'existing', metadata: { type: 'doc' } }, + { id: 'b.md', content: 'existing', metadata: { type: 'doc' } }, + ] + + await indexResources({ ...baseOpts, docsToIndex: docs }) + + expect(listIndexIds).not.toHaveBeenCalled() + expect(createIndex).toHaveBeenCalledWith(docs, expect.objectContaining({ dbPath: expect.any(String) })) + }) + // 6a2: db exists with new docs → incremental index it('incrementally indexes new docs when db exists', async () => { vi.mocked(existsSync).mockReturnValue(true)