feat: optional derived vector index (#49)

.github/workflows/ci.yml

@@ -30,6 +30,24 @@ jobs:
           files: coverage.xml
           fail_ci_if_error: false
 
+  vector:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: astral-sh/setup-uv@v4
+      - run: uv venv
+      - run: uv pip install -e ".[dev,vector]"
+      - run: uv run ruff check src/ tests/
+      - run: uv run ruff format --check src/ tests/
+      - name: Run tests (exercises the Chroma vector index path)
+        run: uv run --extra vector pytest -q
+
   adapters:
     runs-on: ubuntu-latest
     strategy:

CHANGELOG.md

@@ -4,6 +4,41 @@ All notable changes to HotMem will be documented in this file.
 
 Format follows [Keep a Changelog](https://keepachangelog.com/).
 
+## [0.2.4] - 2026-08-28
+
+### Added — Optional derived vector index (#49)
+- Pluggable vector index for search acceleration: `VectorIndex` protocol with
+  a no-op `NullVectorIndex` (default — HotMem runs with zero vector
+  dependencies) and an optional `ChromaVectorIndex` backend
+  (`uv pip install 'hotmem[vector]'`, lazily imported; degrades to the null
+  backend with a warning when chromadb is not installed).
+- The index is disposable and rebuildable: SQLite, files, bundles, and
+  manifests remain canonical storage. Index loss never loses memory —
+  search falls back to the deterministic SQLite cosine+FTS scan whenever
+  the index is absent or stale.
+- Ranking contract preserved: the index only supplies oversampled candidate
+  ids which are re-scored in SQLite with the identical hybrid formula
+  (cosine + FTS + importance), so `/v1/search` responses are identical with
+  or without acceleration (stores larger than the oversample window are
+  approximate by design; the fallback remains the correctness floor).
+- Rebuild reads only SQLite rows (embeddings reused, never recomputed) —
+  backing files are never read; file-backed memories are indexed from
+  eligible inline summaries/metadata only, preserving lazy content reads.
+- Rebuild marker records the store fingerprint (count, max rowid, max event
+  seq) for cheap staleness detection; staleness, missing dependencies, and
+  rebuild state are observable via `GET /v1/vector-index/status`.
+- New admin endpoints: `POST /v1/vector-index/rebuild` (emits an
+  `index.rebuilt` event), `GET /v1/vector-index/status`, and
+  `DELETE /v1/vector-index` (clear). `/v1/search` response shape unchanged.
+- `hotmem serve --vector-index {none,chroma}` CLI flag (default `none`).
+
+### Fixed — JSONL inspector line offsets (#86)
+- `JSONLInspector._stream` computed line offsets in (carry+chunk) coordinates,
+  overstating `unsupported_reason` offsets and `byte_ranges` by the carried
+  byte count whenever an earlier line spanned the 1 MiB read-chunk boundary.
+  Offsets are now exact file coordinates (regression test included; discovered
+  by the native helper spike, #48/#84).
+
 ## [0.2.3] - 2026-08-07
 
 ### Changed

bench/native_spike/gen_corpus.py

@@ -166,9 +166,10 @@ def gen_jsonl_files(out: Path) -> list[dict]:
                 if not bad_written and written >= target * 0.6:
                     bad_line_offset = written
                     bad_line_index = rows
-                    f.write(b'{"id": "broken", "fact_text": "truncated')
+                    bad_line = b'{"id": "broken", "fact_text": "truncated'
+                    f.write(bad_line)
                     f.write(b"\n")
-                    written += 38
+                    written += len(bad_line) + 1
                     rows += 1
                     bad_written = True
                     continue

bench/native_spike/run_bench.py

@@ -48,15 +48,29 @@
 CACHE_MODES = ["warm", "cold"]
 
 
-def spawn(bench: str, arm: str, *, path: str = "", offset: int = 0, length: int = 0,
-          runs: int, cache: str, env: dict | None = None, timeout: int = 1800) -> dict:
+def spawn(
+    bench: str,
+    arm: str,
+    *,
+    path: str = "",
+    offset: int = 0,
+    length: int = 0,
+    runs: int,
+    cache: str,
+    env: dict | None = None,
+    timeout: int = 1800,
+) -> dict:
     cmd = [
         sys.executable,
         str(SPIKE_DIR / "bench_worker.py"),
-        "--bench", bench,
-        "--arm", arm,
-        "--runs", str(runs),
-        "--cache", cache,
+        "--bench",
+        bench,
+        "--arm",
+        arm,
+        "--runs",
+        str(runs),
+        "--cache",
+        cache,
     ]
     if path:
         cmd += ["--path", path, "--offset", str(offset), "--length", str(length)]
@@ -88,6 +102,7 @@ def _ctx() -> dict:
 # b1 — range checksum                                                    #
 # --------------------------------------------------------------------- #
 
+
 def run_b1(manifest: dict, runs: int, quick: bool) -> dict:
     cases = []
     for e in manifest["checksum_files"]:
@@ -123,6 +138,7 @@ def run_b1(manifest: dict, runs: int, quick: bool) -> dict:
 # b2 — JSONL scanning                                                    #
 # --------------------------------------------------------------------- #
 
+
 def run_b2(manifest: dict, runs: int, quick: bool) -> dict:
     files = [e["name"] for e in manifest["jsonl_files"]]
     if quick:
@@ -172,6 +188,7 @@ def run_b2(manifest: dict, runs: int, quick: bool) -> dict:
 # b3 — bundle parse profile                                              #
 # --------------------------------------------------------------------- #
 
+
 def run_b3(manifest: dict, runs: int, quick: bool) -> dict:
     trees = [t["name"] for t in manifest["bundle_trees"]]
     if quick:
@@ -193,6 +210,7 @@ def run_b3(manifest: dict, runs: int, quick: bool) -> dict:
 # b4 — manifest verification                                             #
 # --------------------------------------------------------------------- #
 
+
 def run_b4(runs: int, quick: bool) -> dict:
     print("  [b4] manifest verification …", flush=True)
     cell = {"context": _ctx(), "arms": {}}
@@ -214,6 +232,7 @@ def run_b4(runs: int, quick: bool) -> dict:
 # optional-boundary check (graceful degradation)                         #
 # --------------------------------------------------------------------- #
 
+
 def run_boundary_check() -> dict:
     """Prove helpers degrade gracefully: run a c arm and a wasm arm with
     HOTMEM_SPIKE_DISABLE_NATIVE=1; both must report skipped=helper_unavailable
@@ -222,20 +241,19 @@ def run_boundary_check() -> dict:
     env = {"HOTMEM_SPIKE_DISABLE_NATIVE": "1"}
     binf = benchlib.CORPUS_DIR / "bin_1mb.bin"
     jf = benchlib.CORPUS_DIR / "events_10mb.jsonl"
-    c1 = spawn("b1", "c_pread", path=str(binf), length=binf.stat().st_size,
-               runs=1, cache="warm", env=env)
-    c2 = spawn("b2", "wasm_scan", path=str(jf), runs=1, cache="warm", env=env)
-    ok = (
-        c1.get("skipped") == "helper_unavailable"
-        and c2.get("skipped") == "helper_unavailable"
+    c1 = spawn(
+        "b1", "c_pread", path=str(binf), length=binf.stat().st_size, runs=1, cache="warm", env=env
     )
+    c2 = spawn("b2", "wasm_scan", path=str(jf), runs=1, cache="warm", env=env)
+    ok = c1.get("skipped") == "helper_unavailable" and c2.get("skipped") == "helper_unavailable"
     return {"ok": ok, "c_pread_disabled": c1, "wasm_scan_disabled": c2}
 
 
 # --------------------------------------------------------------------- #
 # report                                                                 #
 # --------------------------------------------------------------------- #
 
+
 def _fmt_ms(v) -> str:
     return "—" if v is None else f"{v:.1f}"
 
@@ -262,18 +280,24 @@ def report(results: dict) -> str:
     lines: list[str] = []
 
     host = results.get("host", {})
-    lines.append(f"Host: {host.get('cpu', '?')} | sha_ni={host.get('sha_ni')} | "
-                 f"kernel {host.get('kernel', '?')} | Python {host.get('python', '?')}")
-    lines.append(f"Profile: {results.get('profile')} | runs={results.get('config', {}).get('runs')}"
-                 f" | corpus seed={results.get('corpus_seed')}")
+    lines.append(
+        f"Host: {host.get('cpu', '?')} | sha_ni={host.get('sha_ni')} | "
+        f"kernel {host.get('kernel', '?')} | Python {host.get('python', '?')}"
+    )
+    lines.append(
+        f"Profile: {results.get('profile')} | runs={results.get('config', {}).get('runs')}"
+        f" | corpus seed={results.get('corpus_seed')}"
+    )
     lines.append("")
 
     # ---- B1 ----
     b1 = results.get("b1_range_checksum", {})
     lines.append("## B1 — Range SHA-256 checksum (median ms, warm / cold; peak RSS MB in parens)")
     lines.append("")
-    header = ("| file (range) | py double-read | py single | py stream | c pread | c mmap "
-              "| py-single gains | c_mmap vs py_single |")
+    header = (
+        "| file (range) | py double-read | py single | py stream | c pread | c mmap "
+        "| py-single gains | c_mmap vs py_single |"
+    )
     lines.append(header)
     lines.append("|" + "---|" * 8)
     for case in b1.get("cases", []):
@@ -312,7 +336,11 @@ def report(results: dict) -> str:
         for arm in B2_ARMS:
             vals.append(f"{_fmt_ms(_med(cell, arm, 'warm'))} / {_fmt_ms(_med(cell, arm, 'cold'))}")
         rows = next(
-            (_arm_cell(cell, a, "warm").get("output", {}).get("rows") for a in B2_ARMS),
+            (
+                out
+                for a in B2_ARMS
+                if (out := _arm_cell(cell, a, "warm").get("output", {}).get("rows")) is not None
+            ),
             None,
         )
         lines.append(f"| {cell['file']} | " + " | ".join(vals) + f" | {rows} |")
@@ -354,23 +382,25 @@ def report(results: dict) -> str:
     # ---- boundary ----
     bc = results.get("optional_boundary_check", {})
     if bc:
-        lines.append("## Optional-boundary check (HOTMEM_SPIKE_DISABLE_NATIVE=1): "
-                     f"**{bc.get('ok')}**")
+        lines.append(
+            f"## Optional-boundary check (HOTMEM_SPIKE_DISABLE_NATIVE=1): **{bc.get('ok')}**"
+        )
         lines.append("")
 
     return "\n".join(lines)
 
 
 # --------------------------------------------------------------------- #
 
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--profile", choices=["reduced", "full"], default="reduced")
-    ap.add_argument("--quick", action="store_true",
-                    help="smoke the harness: 1 run, small inputs")
+    ap.add_argument("--quick", action="store_true", help="smoke the harness: 1 run, small inputs")
     ap.add_argument("--out", default="results.json")
-    ap.add_argument("--report", action="store_true",
-                    help="print markdown report from --out and exit")
+    ap.add_argument(
+        "--report", action="store_true", help="print markdown report from --out and exit"
+    )
     args = ap.parse_args()
 
     out_path = SPIKE_DIR / args.out

docs/api.md

@@ -81,7 +81,46 @@ Content-Type: application/json
 
 Exports all memories to a JSONL or JSONL.GZ swap file.
 
-## 8. OpenAPI Spec
+## 8. Vector Index (optional, derived)
+
+The vector index is an **optional acceleration layer** — it is disposable,
+rebuildable, and **never canonical storage**. SQLite, files, bundles, and
+manifests remain the source of truth. Losing or deleting the index never
+loses memory: search transparently falls back to the deterministic SQLite
+cosine scan, and the index can be fully rebuilt from canonical storage.
+
+Enable it at server start (`hotmem serve --vector-index chroma`, requires the
+optional `hotmem[vector]` extra). The index only supplies candidate ids; final
+ranking is always recomputed with the canonical hybrid scorer, so the
+`/v1/search` response shape and ranking are identical with or without
+acceleration.
+
+```http
+POST /v1/vector-index/rebuild
+```
+
+Rebuilds the index from SQLite rows (embeddings are reused, never recomputed).
+Reads no backing files. Returns `indexed_count`, `db_count`,
+`skipped_no_embedding`, `rebuilt_at`, and `trace_ms`. Returns
+`400 vector_index_disabled` when no backend is configured, or
+`400 vector_dependency_missing` when the backend package is not installed.
+Emits an `index.rebuilt` event.
+
+```http
+GET /v1/vector-index/status
+```
+
+Returns `backend`, `requested_backend`, `dependency_available`,
+`indexed_count`, `db_count`, `stale`, `rebuilt_at`, `path`, and `trace_ms`.
+`stale: true` means search is currently served by the deterministic fallback.
+
+```http
+DELETE /v1/vector-index
+```
+
+Clears the index (entries + rebuild marker). Canonical storage is untouched.
+
+## 9. OpenAPI Spec
 
 Export the machine-readable spec:
 

docs/architecture.md

@@ -34,6 +34,13 @@ HotMem combines deterministic text embeddings, keyword overlap, and importance
 to rank local memories. Read-only inspectors provide lightweight metadata for
 CSV, JSONL, and Parquet files without turning the runtime into a query engine.
 
+An optional derived vector index can accelerate candidate retrieval. The index
+is disposable and rebuildable from SQLite, sits in front of the canonical
+hybrid ranker rather than replacing it, and search falls back to the
+deterministic SQLite scan whenever the index is absent or stale. SQLite, files,
+bundles, and manifests remain canonical storage; the index is never a source
+of truth.
+
 ## Portability
 
 JSONL and JSONL.GZ are supported portable record formats. Snapshot v2 adds a

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hotmem"
-version = "0.2.3"
+version = "0.2.4"
 description = "A local-first memory sidecar for agent applications"
 readme = "README.md"
 requires-python = ">=3.11"
@@ -38,6 +38,9 @@ dev = [
 docs = [
     "mkdocs-material>=9,<10",
 ]
+vector = [
+    "chromadb>=1.0,<2",
+]
 
 [project.scripts]
 hotmem = "hotmem.cli:main"

src/hotmem/__init__.py

@@ -1,3 +1,3 @@
 """HotMem — A local-first memory sidecar for agent applications."""
 
-__version__ = "0.2.3"
+__version__ = "0.2.4"

src/hotmem/cli.py

@@ -38,7 +38,15 @@ def main():
 @click.option("--mount", default=None, type=click.Path(), help="Mount directory path.")
 @click.option("--db", "db_path", default=None, type=click.Path(), help="Explicit database path.")
 @click.option("--host", default="127.0.0.1", help="Host to bind to.")
-def serve(port: int, mount: str | None, db_path: str | None, host: str):
+@click.option(
+    "--vector-index",
+    "vector_backend",
+    default="none",
+    type=click.Choice(["none", "chroma"]),
+    help="Optional derived vector index backend (default: none). The index is "
+    "disposable and rebuildable; SQLite remains canonical storage.",
+)
+def serve(port: int, mount: str | None, db_path: str | None, host: str, vector_backend: str):
     """Start the HotMem sidecar server."""
     import uvicorn
 
@@ -59,7 +67,12 @@ def serve(port: int, mount: str | None, db_path: str | None, host: str):
             detail={"path": db_path},
         )
 
-    app = create_app(db_path=db_path, swap_path=swap_path, port=port)
+    app = create_app(
+        db_path=db_path,
+        swap_path=swap_path,
+        port=port,
+        vector_backend=vector_backend,
+    )
 
     _trace.info(
         "serve",