diff --git a/.changeset/sqlite-vec-targets.md b/.changeset/sqlite-vec-targets.md new file mode 100644 index 0000000..8250dc5 --- /dev/null +++ b/.changeset/sqlite-vec-targets.md @@ -0,0 +1,5 @@ +--- +"antisprawl": patch +--- + +Embed sqlite-vec loadables for all five CLI release targets. diff --git a/docs/architecture.md b/docs/architecture.md index 4ab8c8e..6eb22d5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -325,7 +325,7 @@ It does not store Findings, Finding evidence, Finding outcomes, or delivery hist It stores no source, prompts, commands, transcripts, or credentials. -The Index uses one canonical ordinary float32 vector BLOB representation for both search paths. On Linux x64, the executable embeds pinned `asg017/sqlite-vec` v0.1.9, verifies it, atomically extracts it to a private versioned cache, loads it on the query connection, and probes its version and cosine function. sqlite-vec retrieves candidates within a dimension-derived float32 error margin of the semantic threshold. Application full-precision cosine rescoring of those candidates remains authoritative for gates, public evidence, and ranking. Inputs outside the margin's safe numeric range, or any digest, extraction, load, probe, or query failure, switch the whole Edit batch to application exact search over the same BLOBs without reindexing or changing Coverage. +The Index uses one canonical ordinary float32 vector BLOB representation for both search paths. Each linux-x64, linux-arm64, darwin-x64, darwin-arm64, and windows-x64 executable embeds only its pinned `asg017/sqlite-vec` v0.1.9 loadable, verifies it, atomically extracts it to a private versioned cache, loads it on the query connection, and probes its version and cosine function. sqlite-vec retrieves candidates within a dimension-derived float32 error margin of the semantic threshold. Application full-precision cosine rescoring of those candidates remains authoritative for gates, public evidence, and ranking. Inputs outside the margin's safe numeric range, or any digest, extraction, load, probe, or query failure, switch the whole Edit batch to application exact search over the same BLOBs without reindexing or changing Coverage. Quantized search requires recall benchmarks before enablement. ANN indexes are a v1 non-goal. LanceDB, USearch, and alternate storage engines are deferred until measured scale requires them. @@ -514,7 +514,6 @@ The architecture deliberately does not fix values that must come from evidence: - embedding candidate count; - 384 versus 1536 dimensions; - watcher debounce and reconciliation intervals; -- sqlite-vec target support; - whether quantized search preserves adequate recall. These values are versioned once selected and remain visible in configuration or index provenance. diff --git a/packages/cli/build.ts b/packages/cli/build.ts new file mode 100644 index 0000000..287ee76 --- /dev/null +++ b/packages/cli/build.ts @@ -0,0 +1,26 @@ +import provenance from "./vendor/sqlite-vec/provenance.json" with { type: "json" }; +import { sqliteVecHost } from "./src/sqlite-vec-target.ts"; + +const target = sqliteVecHost(process.platform, process.arch); + +const record = target === undefined ? undefined : provenance.libraries[target]; + +if (record === undefined) { + throw new Error(`sqlite-vec is not vendored for ${process.platform}/${process.arch}`); +} + +const result = Bun.spawnSync( + [ + "bun", + "build", + "--compile", + "--asset", + `vendor/sqlite-vec/${record.library}`, + "src/main.ts", + "--outfile", + "dist/antisprawl", + ], + { stderr: "inherit", stdout: "inherit" }, +); + +process.exit(result.exitCode ?? 1); diff --git a/packages/cli/package.json b/packages/cli/package.json index dd6bbf0..420cf68 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -5,7 +5,7 @@ "description": "Keep your agent from writing spaghetti code.", "type": "module", "scripts": { - "build": "bun build --compile src/main.ts --outfile dist/antisprawl" + "build": "bun build.ts" }, "dependencies": { "@effect/platform-bun": "4.0.0-rc.115", diff --git a/packages/cli/src/assets.d.ts b/packages/cli/src/assets.d.ts index e70c1ee..03e334f 100644 --- a/packages/cli/src/assets.d.ts +++ b/packages/cli/src/assets.d.ts @@ -17,3 +17,13 @@ declare module "*.so" { const path: string; export default path; } + +declare module "*.dylib" { + const path: string; + export default path; +} + +declare module "*.dll" { + const path: string; + export default path; +} diff --git a/packages/cli/src/sqlite-vec-target.ts b/packages/cli/src/sqlite-vec-target.ts new file mode 100644 index 0000000..9de0458 --- /dev/null +++ b/packages/cli/src/sqlite-vec-target.ts @@ -0,0 +1,14 @@ +export const sqliteVecHost = (platform: string, arch: string) => { + switch (platform === "win32" ? `windows-${arch}` : `${platform}-${arch}`) { + case "linux-x64": + return "linux-x64"; + case "linux-arm64": + return "linux-arm64"; + case "darwin-x64": + return "darwin-x64"; + case "darwin-arm64": + return "darwin-arm64"; + case "windows-x64": + return "windows-x64"; + } +}; diff --git a/packages/cli/src/vector-search.ts b/packages/cli/src/vector-search.ts index a8b8411..7ce40ac 100644 --- a/packages/cli/src/vector-search.ts +++ b/packages/cli/src/vector-search.ts @@ -4,12 +4,13 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; -import sqliteVecAsset from "../vendor/sqlite-vec/linux-x64/vec0.so" with { type: "file" }; +import provenance from "../vendor/sqlite-vec/provenance.json" with { type: "json" }; import { appError } from "./errors.ts"; +import { sqliteVecHost } from "./sqlite-vec-target.ts"; const version = "v0.1.9"; -const librarySha256 = "5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8"; +export { sqliteVecHost }; const ProbeRow = Schema.Struct({ version: Schema.Literal(version), distance: Schema.Finite }); @@ -75,11 +76,21 @@ const sha256 = (bytes: Uint8Array) => Bun.CryptoHasher.hash("sha256", bytes, "he const extractedLibrary = Effect.fn("VectorSearch.extract")(function* ( environment: Readonly>, ) { - if (process.platform !== "linux" || process.arch !== "x64") return yield* safeError(); + const target = sqliteVecHost(process.platform, process.arch); + const record = target === undefined ? undefined : provenance.libraries[target]; + + if (target === undefined || record === undefined) return yield* safeError(); const fs = yield* FileSystem.FileSystem; const paths = yield* Path.Path; - const bundled = yield* fs.readFile(sqliteVecAsset); + const librarySha256 = record.librarySha256; + const filename = record.library.slice(record.library.lastIndexOf("/") + 1); + + const bundled = yield* fs.readFile( + Bun.isStandaloneExecutable + ? paths.join(import.meta.dir, filename) + : paths.join(import.meta.dir, "../vendor/sqlite-vec", record.library), + ); if ( environment.ANTISPRAW_ACCEPTANCE_VECTOR_SEARCH_FAILURE === "digest" || @@ -94,8 +105,8 @@ const extractedLibrary = Effect.fn("VectorSearch.extract")(function* ( const cacheRoot = environment.XDG_CACHE_HOME ?? paths.join(homedir(), ".cache"); const appDirectory = paths.join(cacheRoot, "antisprawl"); - const directory = paths.join(appDirectory, `sqlite-vec-${version}-linux-x64-${librarySha256}`); - const library = paths.join(directory, "vec0.so"); + const directory = paths.join(appDirectory, `sqlite-vec-${version}-${target}-${librarySha256}`); + const library = paths.join(directory, filename); yield* fs.makeDirectory(appDirectory, { recursive: true, mode: 0o700 }); yield* fs.chmod(appDirectory, 0o700); @@ -106,7 +117,10 @@ const extractedLibrary = Effect.fn("VectorSearch.extract")(function* ( const valid = existing && sha256(yield* fs.readFile(library)) === librarySha256; if (!valid) { - const temporary = paths.join(directory, `.vec0-${process.pid}-${Bun.randomUUIDv7()}.so`); + const temporary = paths.join( + directory, + `.vec0-${process.pid}-${Bun.randomUUIDv7()}${filename.slice(filename.lastIndexOf("."))}`, + ); yield* fs .writeFile(temporary, bundled) diff --git a/packages/cli/tests/acceptance/index-compiled.bun.test.ts b/packages/cli/tests/acceptance/index-compiled.bun.test.ts index 426cb26..3f69490 100644 --- a/packages/cli/tests/acceptance/index-compiled.bun.test.ts +++ b/packages/cli/tests/acceptance/index-compiled.bun.test.ts @@ -39,6 +39,8 @@ compiledTest( "build", "--compile", "--target=bun-linux-x64", + "--asset", + "vendor/sqlite-vec/linux-x64/vec0.so", sourceEntrypoint, "--outfile", executable, diff --git a/packages/cli/tests/acceptance/openai-live.bun.test.ts b/packages/cli/tests/acceptance/openai-live.bun.test.ts index 6f9ae3c..942e5a8 100644 --- a/packages/cli/tests/acceptance/openai-live.bun.test.ts +++ b/packages/cli/tests/acceptance/openai-live.bun.test.ts @@ -49,6 +49,8 @@ liveTest( "build", "--compile", "--target=bun-linux-x64", + "--asset", + "vendor/sqlite-vec/linux-x64/vec0.so", "--define", "globalThis.ANTISPRAW_LIVE_PROFILE_MATRIX=true", sourceEntrypoint, diff --git a/packages/cli/tests/bun/vector-search.test.ts b/packages/cli/tests/bun/vector-search.test.ts index 907d9d0..1ec393f 100644 --- a/packages/cli/tests/bun/vector-search.test.ts +++ b/packages/cli/tests/bun/vector-search.test.ts @@ -7,7 +7,8 @@ import * as Path from "effect/Path"; import { nativeSearch } from "../../src/app.ts"; import { detectProbableDuplicates, type IndexedSymbol } from "../../src/detector.ts"; import { configuredEmbeddingProvider } from "../../src/embedding.ts"; -import { searchNativeCandidates } from "../../src/vector-search.ts"; +import provenance from "../../vendor/sqlite-vec/provenance.json" with { type: "json" }; +import { searchNativeCandidates, sqliteVecHost } from "../../src/vector-search.ts"; const run = (effect: Effect.Effect) => Effect.runPromise(effect.pipe(Effect.provide(BunServices.layer))); @@ -131,9 +132,16 @@ test("sqlite-vec retrieves all ordinary vector BLOB candidates by cosine", () => const appDirectory = paths.join(cachePath, "antisprawl"); const [versionDirectory] = yield* fs.readDirectory(appDirectory); + const host = sqliteVecHost(process.platform, process.arch); + const record = host === undefined ? undefined : provenance.libraries[host]; + const filename = record?.library.slice(record.library.lastIndexOf("/") + 1); + + expect(host).toBeDefined(); + expect(record).toBeDefined(); + expect(versionDirectory).toContain(`sqlite-vec-v0.1.9-${host}-`); expect((yield* fs.stat(appDirectory)).mode & 0o777).toBe(0o700); expect( - (yield* fs.stat(paths.join(appDirectory, versionDirectory!, "vec0.so"))).mode & 0o777, + (yield* fs.stat(paths.join(appDirectory, versionDirectory!, filename!))).mode & 0o777, ).toBe(0o600); }), ), @@ -209,6 +217,41 @@ test("native search wiring preserves boundary Findings while reducing rescoring" ); }); +test("sqlite-vec host mapping covers the five CLI targets", () => { + expect(sqliteVecHost("linux", "x64")).toBe("linux-x64"); + expect(sqliteVecHost("linux", "arm64")).toBe("linux-arm64"); + expect(sqliteVecHost("darwin", "x64")).toBe("darwin-x64"); + expect(sqliteVecHost("darwin", "arm64")).toBe("darwin-arm64"); + expect(sqliteVecHost("win32", "x64")).toBe("windows-x64"); + expect(sqliteVecHost("win32", "arm64")).toBeUndefined(); + expect(sqliteVecHost("freebsd", "x64")).toBeUndefined(); + expect(Object.keys(provenance.libraries).sort()).toEqual([ + "darwin-arm64", + "darwin-x64", + "linux-arm64", + "linux-x64", + "windows-x64", + ]); +}); + +test("vendored sqlite-vec loadables match provenance digests", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const paths = yield* Path.Path; + + for (const record of Object.values(provenance.libraries)) { + const bytes = yield* fs.readFile( + paths.join(import.meta.dir, "../../vendor/sqlite-vec", record.library), + ); + + expect(Bun.CryptoHasher.hash("sha256", bytes, "hex"), record.library).toBe( + record.librarySha256, + ); + } + }), + )); + test("native digest, extraction, load, and probe failures stay safe", () => run( Effect.scoped( diff --git a/packages/cli/vendor/sqlite-vec/darwin-arm64/vec0.dylib b/packages/cli/vendor/sqlite-vec/darwin-arm64/vec0.dylib new file mode 100644 index 0000000..1605e79 Binary files /dev/null and b/packages/cli/vendor/sqlite-vec/darwin-arm64/vec0.dylib differ diff --git a/packages/cli/vendor/sqlite-vec/darwin-x64/vec0.dylib b/packages/cli/vendor/sqlite-vec/darwin-x64/vec0.dylib new file mode 100644 index 0000000..b48250c Binary files /dev/null and b/packages/cli/vendor/sqlite-vec/darwin-x64/vec0.dylib differ diff --git a/packages/cli/vendor/sqlite-vec/linux-arm64/vec0.so b/packages/cli/vendor/sqlite-vec/linux-arm64/vec0.so new file mode 100644 index 0000000..536702c Binary files /dev/null and b/packages/cli/vendor/sqlite-vec/linux-arm64/vec0.so differ diff --git a/packages/cli/vendor/sqlite-vec/provenance.json b/packages/cli/vendor/sqlite-vec/provenance.json index c61e947..571e904 100644 --- a/packages/cli/vendor/sqlite-vec/provenance.json +++ b/packages/cli/vendor/sqlite-vec/provenance.json @@ -2,11 +2,42 @@ "repository": "https://github.com/asg017/sqlite-vec", "version": "v0.1.9", "commit": "e9f598abfa0c06b328d8fe5da9c3760cce74be10", - "artifact": "sqlite-vec-0.1.9-loadable-linux-x86_64.tar.gz", - "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-linux-x86_64.tar.gz", - "artifactSha256": "b959baa1d8dc88861b1edb337b8587178cdcb12d60b4998f9d10b6a82052d5d7", - "target": "linux-x86_64", - "library": "linux-x64/vec0.so", - "librarySha256": "5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8", - "entrypoint": "sqlite3_vec_init" + "entrypoint": "sqlite3_vec_init", + "libraries": { + "linux-x64": { + "artifact": "sqlite-vec-0.1.9-loadable-linux-x86_64.tar.gz", + "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-linux-x86_64.tar.gz", + "artifactSha256": "b959baa1d8dc88861b1edb337b8587178cdcb12d60b4998f9d10b6a82052d5d7", + "library": "linux-x64/vec0.so", + "librarySha256": "5923730861b86c707cca5602b5f91092f9e52a46706dbc6e269fd4bb9c4498e8" + }, + "linux-arm64": { + "artifact": "sqlite-vec-0.1.9-loadable-linux-aarch64.tar.gz", + "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-linux-aarch64.tar.gz", + "artifactSha256": "ea03d39541e478fab5974253c461e1cb5d77742f69e40cf96e3fad5bc309a37c", + "library": "linux-arm64/vec0.so", + "librarySha256": "0b84cbd06418ca3040827deddd650539be05be0f657952426b926c8606217437" + }, + "darwin-x64": { + "artifact": "sqlite-vec-0.1.9-loadable-macos-x86_64.tar.gz", + "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-macos-x86_64.tar.gz", + "artifactSha256": "53ad76e400786515e2edcaed2f01271dda846316390b761fadbd2dcf56aa4713", + "library": "darwin-x64/vec0.dylib", + "librarySha256": "d39d33d16d302d57440d9a9cdbe3cc095e8324aa96b979b5171f545a6346996d" + }, + "darwin-arm64": { + "artifact": "sqlite-vec-0.1.9-loadable-macos-aarch64.tar.gz", + "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-macos-aarch64.tar.gz", + "artifactSha256": "8282126333399ddfe98bbbcc7a1936e7252625aac49df056a98be602e46bfd29", + "library": "darwin-arm64/vec0.dylib", + "librarySha256": "193e480c50b59a55977d166f4aaf0e1bc8832d6963516e5950f39e4d2ce0b793" + }, + "windows-x64": { + "artifact": "sqlite-vec-0.1.9-loadable-windows-x86_64.tar.gz", + "artifactUrl": "https://github.com/asg017/sqlite-vec/releases/download/v0.1.9/sqlite-vec-0.1.9-loadable-windows-x86_64.tar.gz", + "artifactSha256": "51581189d52066b4dfc6631f6d7a3eab7dedc2260656ab09ca97ab3fb8165983", + "library": "windows-x64/vec0.dll", + "librarySha256": "fcf98662a7ad9dce394b96a88f91032047823831b951c76636787c312a6476e6" + } + } } diff --git a/packages/cli/vendor/sqlite-vec/windows-x64/vec0.dll b/packages/cli/vendor/sqlite-vec/windows-x64/vec0.dll new file mode 100644 index 0000000..83a6749 Binary files /dev/null and b/packages/cli/vendor/sqlite-vec/windows-x64/vec0.dll differ diff --git a/scripts/release.ts b/scripts/release.ts index f96a13c..350cc26 100644 --- a/scripts/release.ts +++ b/scripts/release.ts @@ -2,6 +2,8 @@ import { spawnSync } from "node:child_process"; import { chmodSync, copyFileSync, cpSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { sqliteVecHost } from "../packages/cli/src/sqlite-vec-target.ts"; +import provenance from "../packages/cli/vendor/sqlite-vec/provenance.json" with { type: "json" }; import cliPackage from "../packages/cli/package.json" with { type: "json" }; import piPackage from "../packages/pi/package.json" with { type: "json" }; @@ -133,10 +135,23 @@ const main = () => { const archiveName = `antisprawl-v${cliVersion}-${archiveOs(target.os)}-${target.cpu}.${target.archive}`; const archivePath = join(archives, archiveName); + const vecTarget = sqliteVecHost(target.os, target.cpu); + const vecLibrary = vecTarget === undefined ? undefined : provenance.libraries[vecTarget]; + + if (vecLibrary === undefined) throw new Error(`missing sqlite-vec for ${target.bun}`); + mkdirSync(pkgDir, { recursive: true }); run( "bun", - ["build", "--compile", `--target=${target.bun}`, `--outfile=${outfile}`, entry], + [ + "build", + "--compile", + `--target=${target.bun}`, + "--asset", + join(root, "packages/cli/vendor/sqlite-vec", vecLibrary.library), + `--outfile=${outfile}`, + entry, + ], root, ); chmodSync(outfile, 0o755);