From 6918fb0682f840cf7e6fd5a3095f77d8985c7e86 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Fri, 23 Oct 2026 21:17:16 -0400 Subject: [PATCH] GH-36: refresh the index when store files change --- .agents/decisions.yaml | 23 +++- .agents/subsystems.yaml | 18 +-- README.md | 7 +- hook/match_test.go | 23 ++++ hook/search.go | 19 +-- install/skill.go | 2 +- knowledge/store.go | 19 +++ knowledge/store_test.go | 35 ++++++ main.go | 11 +- search/index.go | 227 +++++++++++++++++++++++++++++++---- search/refresh_test.go | 254 ++++++++++++++++++++++++++++++++++++++++ tools/registry.go | 17 ++- tools/registry_test.go | 1 + tools/reindex.go | 28 +++++ tools/reindex_test.go | 122 +++++++++++++++++++ 15 files changed, 746 insertions(+), 60 deletions(-) create mode 100644 search/refresh_test.go create mode 100644 tools/reindex.go create mode 100644 tools/reindex_test.go diff --git a/.agents/decisions.yaml b/.agents/decisions.yaml index 3786242..fa3c212 100644 --- a/.agents/decisions.yaml +++ b/.agents/decisions.yaml @@ -2,7 +2,7 @@ project: knowledge-mcp version: 1 entries: - id: dec-001 - summary: "Org-level knowledge is indexed per markdown section, not as whole files" + summary: Org-level knowledge is indexed per markdown section, not as whole files detail: |- Whole-file indexing made org queries return 5-6K tokens of unrelated content (measured 2026-09-08: "daisy test-stability" on chaon-io returned full architecture.md and review.md). Split .agents/knowledge/*.md on "## " headings; each section is its own document with key org-::. @@ -11,7 +11,7 @@ entries: source: user-requested token-savings fix, 2026-09-08; GH-34 update, 2026-10-07 date: "2026-10-07" - id: dec-002 - summary: "Merged global entries carry explicit (global) provenance in list output" + summary: Merged global entries carry explicit (global) provenance in list output detail: |- list_knowledge for any non-global project prepends _global entries; previously they were indistinguishable from the project's own entries. Section headers now carry a " (global)" suffix; projects with no local entries get an explicit "no project-specific knowledge entries found" note. @@ -19,7 +19,7 @@ entries: source: user-requested provenance fix, 2026-09-08 date: "2026-09-08" - id: dec-003 - summary: "Query text is escaped so special characters are always literal" + summary: Query text is escaped so special characters are always literal detail: |- User queries are plain text, not lucene syntax; characters like : ( ) && || " were previously interpreted as operators. escapeQueryString escapes syntax-active characters (context-aware for +/- and &&/||); mid-word hyphens like test-stability stay intact. @@ -27,11 +27,24 @@ entries: source: user-requested robustness fix, 2026-09-08 date: "2026-09-08" - id: dec-004 - summary: "Startup indexing: always built in memory (no skip flag)" + summary: 'Startup indexing: always built in memory (no skip flag)' detail: |- The former background-indexing flag --no-index-on-startup is now a deprecated no-op (accepted, warns, ignored). The index is always built in memory at startup; queries block on the ready channel until IndexAll completes. readyOnce (sync.Once) prevents double-close panics. Recorded so a skip-indexing flag is not reintroduced. source: GH-34 memory-only index, 2026-10-07 - date: "2026-10-07" \ No newline at end of file + date: "2026-10-07" + - id: dec-005 + summary: Stale index fixed by fingerprint auto-refresh plus a forced reindex tool + detail: |- + Issue #36: the index was built once at startup, so direct store edits were visible to list_knowledge but not query_knowledge, silently. + Chosen fix (user-approved): build-then-swap on staleness, plus an explicit reindex_knowledge tool. + Why not re-run IndexAll as-is: Add is an upsert keyed by project/id and bleve has no Delete wired up, so entries removed from the files would keep matching. A fresh index per rebuild drops them naturally. + Fingerprint = sha256 over each ref's address/kind and each source file's path, size and mtime; missing files are part of the stamp so creation and deletion both trigger a rebuild. + Query computes the fingerprint before searching and rebuilds under refreshMu; the swap takes the write lock so in-flight queries finish on the old index before it is closed. + Add does not update the fingerprint: the next query rebuilds after a tool write, which is slightly wasteful but cannot swallow a concurrent out-of-band change. + Org knowledge now indexes every top-level *.md in .agents/knowledge/ (shared helper knowledge.OrgKnowledgeFiles) instead of the architecture.md/review.md allowlist, in both the index and the Grep/Glob hook. + list_knowledge stays file-backed for projects and index-backed for orgs; with auto-refresh both are current. + source: 'issue #36 design discussion with user, 2026-10-23' + date: "2026-10-23" diff --git a/.agents/subsystems.yaml b/.agents/subsystems.yaml index 3bce612..ec6ccf4 100644 --- a/.agents/subsystems.yaml +++ b/.agents/subsystems.yaml @@ -2,19 +2,21 @@ project: knowledge-mcp version: 1 entries: - id: sub-001 - summary: "Search index: per-process in-memory bleve, scoped keys, org sections" + summary: 'Search index: in-memory bleve, auto-refresh on store edits, scoped keys' detail: |- The index is built in memory (bleve.NewMemOnly) at server startup from the store YAML/MD files — every restart re-indexes. There is no disk state, no index lock, and any number of instances may query concurrently. + A store fingerprint (per-file path, size, mtime) is recorded at build time; Query recomputes it and rebuilds into a fresh index when the files changed, so direct edits, git checkouts and migration scripts are searchable without a restart. + Rebuild is build-then-swap under an RWMutex: queries in flight finish on the old index, then it is closed. There is no Delete API, so deletes vanish only because the replacement index is built from scratch. + reindex_knowledge forces a rebuild; Refresh reports whether the fingerprint changed. Documents carry summary, detail, rule, category, project with scoped keys /. - Query results de-scope the ID. - Org-level docs (.agents/knowledge/*.md) are split on "## " headings with keys org-::. + Query results de-scope the ID. Org-level docs: every top-level *.md in .agents/knowledge/ (knowledge.OrgKnowledgeFiles), split on "## " headings with keys org-::. User query text is escaped (escapeQueryString) so special characters are literal, never lucene syntax. --index and --no-index-on-startup are accepted warning-logged no-ops. source: GH-34 memory-only index, 2026-10-07 - date: "2026-10-07" + date: "2026-10-23" - id: sub-002 - summary: "Project resolver: kinds, addressing, dynamic fallback" + summary: 'Project resolver: kinds, addressing, dynamic fallback' detail: |- projects/resolver.go builds refs: KindProject (has .agents/ yaml), KindOrg (root with .agents/knowledge/), KindGlobal (--global path, addressed _global). Roots scan immediate children only, skipping .agents and .git; symlinks followed; unreadable entries warn. @@ -23,7 +25,7 @@ entries: source: code review, 2026-09-08; GH-34 update, 2026-10-07 date: "2026-10-07" - id: sub-003 - summary: "Knowledge store: atomic writes, per-file locks, entry lifecycle" + summary: 'Knowledge store: atomic writes, per-file locks, entry lifecycle' detail: |- knowledge/store.go: Save writes temp file, fsyncs, renames (atomic); LoadOrCreate returns empty KnowledgeFile when missing. tools package keeps per-path RWMutexes (fileLocks) around read-modify-write cycles; write/update lock, list takes RLock. @@ -34,7 +36,7 @@ entries: source: code review, 2026-09-08 date: "2026-09-08" - id: sub-004 - summary: "Startup indexing: always-on background kick, no skip flag" + summary: 'Startup indexing: always-on background kick, no skip flag' detail: |- IndexAll always runs in a background goroutine at startup (there is no skip flag). Queries block on the ready channel until IndexAll completes. @@ -42,4 +44,4 @@ entries: --no-index-on-startup and --index are deprecated, accepted, warn, and do nothing. The index lives only as long as the process and is rebuilt from the store files on every startup. source: GH-34 memory-only index, 2026-10-07 - date: "2026-10-07" \ No newline at end of file + date: "2026-10-07" diff --git a/README.md b/README.md index 2442836..ed1c56a 100644 --- a/README.md +++ b/README.md @@ -177,10 +177,14 @@ General notes: - Each `--root` discovers its immediate children as projects (one level deep — pass deeper directories as additional flags). - Duplicate project basenames across roots are addressed as `/` (e.g. `work/api`); `list_projects` shows which names need qualification. -- `query_knowledge` accepts org root names to search that root's `.agents/knowledge/` files. Org-level documents are indexed **per `##` section**, so a query returns the matching section(s), not the entire file. +- `query_knowledge` accepts org root names to search that root's `.agents/knowledge/` files. Every `.md` file in that directory is indexed (not a fixed filename allowlist), and documents are indexed **per `##` section**, so a query returns the matching section(s), not the entire file. - The search index is built **in memory** per server process at startup and does not survive a restart. There is no index on disk anywhere, no lock, and no contention — any number of instances may query concurrently. +- The index notices out-of-band store edits (hand edits, `git checkout`, + migration scripts) and rebuilds itself before serving the next query, so + `query_knowledge` never silently disagrees with `list_knowledge`. Use + `reindex_knowledge` to force a full rebuild. - `--index ` is deprecated and ignored (the search index is in-memory). - Old leftover `.agents/.index/` and `/.index/` directories and `$XDG_STATE_HOME/knowledge-mcp/` artifacts from earlier releases are inert and can be deleted. @@ -222,6 +226,7 @@ arguments — entry bodies are replaced by sizes and never written. Set | `list_knowledge` | List all entries; entries with rules shown first as constraints | | `update_knowledge` | Update an existing entry by ID | | `list_projects` | List all discovered projects (org-wide mode only) | +| `reindex_knowledge` | Rebuild the search index from the store files (queries also refresh automatically) | Queries are plain-text search with no operator syntax — special characters in queries are always treated as literal text. diff --git a/hook/match_test.go b/hook/match_test.go index edf74b2..50e7938 100644 --- a/hook/match_test.go +++ b/hook/match_test.go @@ -105,6 +105,29 @@ func TestSearchIncludesOrgAndGlobal(t *testing.T) { } } +// TestSearchIncludesAdditionalOrgMarkdown covers the org knowledge glob: +// every markdown file in .agents/knowledge/ is searched, not just the +// historical architecture.md/review.md pair. +func TestSearchIncludesAdditionalOrgMarkdown(t *testing.T) { + root := t.TempDir() + orgKnowledge := filepath.Join(root, ".agents", "knowledge") + if err := os.MkdirAll(orgKnowledge, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(orgKnowledge, "notes.md"), + []byte("## Takedown\npurge_requested_at drives takedown\n"), 0644); err != nil { + t.Fatal(err) + } + res, _, err := projects.Build([]string{root}, nil, "") + if err != nil { + t.Fatal(err) + } + hits := Search(res, nil, orgRef(t, res), "takedown", 3, nil) + if len(hits) != 1 || hits[0].ID != "org-notes.md::Takedown" { + t.Fatalf("hits = %+v", hits) + } +} + func orgRef(t *testing.T, res *projects.Resolver) *projects.Ref { t.Helper() for _, ref := range res.Snapshot() { diff --git a/hook/search.go b/hook/search.go index 0148e4a..a40f1e7 100644 --- a/hook/search.go +++ b/hook/search.go @@ -45,14 +45,17 @@ func Search(res *projects.Resolver, proj, org *projects.Ref, pattern string, lim } } if org != nil { - for _, file := range []string{"architecture.md", "review.md"} { - data, err := os.ReadFile(filepath.Join(res.OrgKnowledgeDir(*org), file)) - if err != nil { - continue - } - files++ - for _, sec := range knowledge.SplitSections(string(data)) { - addEntry(org.Address, "org-"+file+"::"+sec[0], file+": "+sec[0], sec[1], "") + names, err := knowledge.OrgKnowledgeFiles(res.OrgKnowledgeDir(*org)) + if err == nil { + for _, file := range names { + data, readErr := os.ReadFile(filepath.Join(res.OrgKnowledgeDir(*org), file)) + if readErr != nil { + continue + } + files++ + for _, sec := range knowledge.SplitSections(string(data)) { + addEntry(org.Address, "org-"+file+"::"+sec[0], file+": "+sec[0], sec[1], "") + } } } } diff --git a/install/skill.go b/install/skill.go index 1c01f69..cfd449e 100644 --- a/install/skill.go +++ b/install/skill.go @@ -14,7 +14,7 @@ func RenderSkillMD(registry []tools.ToolSpec) []byte { } md := "---\n" + "name: knowledge-mcp\n" + - "description: \"Use the persistent knowledge store for project conventions, subsystems, decisions, and workflows. Triggers on: what are our conventions, does this project have rules about, how does this subsystem work, why was this decision made, run the brainstorm workflow, run the plan workflow, debug workflow, verify workflow, capture this decision, write that down as knowledge, update that entry, knowledge store, .agents, list_knowledge, query_knowledge, write_knowledge, update_knowledge, init_knowledge.\"\n" + + "description: \"Use the persistent knowledge store for project conventions, subsystems, decisions, and workflows. Triggers on: what are our conventions, does this project have rules about, how does this subsystem work, why was this decision made, run the brainstorm workflow, run the plan workflow, debug workflow, verify workflow, capture this decision, write that down as knowledge, update that entry, knowledge store, .agents, list_knowledge, query_knowledge, write_knowledge, update_knowledge, init_knowledge, reindex_knowledge.\"\n" + "---\n" + "\n" + "# Knowledge Store (knowledge-mcp)\n" + diff --git a/knowledge/store.go b/knowledge/store.go index 133ac6f..61290f0 100644 --- a/knowledge/store.go +++ b/knowledge/store.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strconv" "strings" @@ -118,6 +119,24 @@ func LoadOrCreate(filePath, project string) (*KnowledgeFile, error) { return kf, nil } +// OrgKnowledgeFiles returns the sorted names of the markdown files directly +// under an org knowledge directory. Subdirectories are not walked. +func OrgKnowledgeFiles(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var files []string + for _, entry := range entries { + if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".md") { + continue + } + files = append(files, entry.Name()) + } + sort.Strings(files) + return files, nil +} + // SplitSections splits a knowledge markdown document on lines starting // with "## ". Content before the first section heading is kept under // "Overview". Subsection headings (### ...) stay part of their parent diff --git a/knowledge/store_test.go b/knowledge/store_test.go index 17ad76e..20341bf 100644 --- a/knowledge/store_test.go +++ b/knowledge/store_test.go @@ -170,3 +170,38 @@ func TestEnsureDir(t *testing.T) { t.Fatalf("EnsureDir() second call error: %v", err) } } + +func TestOrgKnowledgeFiles(t *testing.T) { + dir := t.TempDir() + for _, name := range []string{"review.md", "architecture.md", "notes.md", "ignored.txt"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("## Section\nbody\n"), 0644); err != nil { + t.Fatalf("WriteFile(%s) error: %v", name, err) + } + } + if err := os.Mkdir(filepath.Join(dir, "nested"), 0755); err != nil { + t.Fatalf("Mkdir() error: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "nested", "deep.md"), []byte("## Deep\nbody\n"), 0644); err != nil { + t.Fatalf("WriteFile(nested) error: %v", err) + } + + files, err := OrgKnowledgeFiles(dir) + if err != nil { + t.Fatalf("OrgKnowledgeFiles() error: %v", err) + } + want := []string{"architecture.md", "notes.md", "review.md"} + if len(files) != len(want) { + t.Fatalf("OrgKnowledgeFiles() = %v, want %v", files, want) + } + for i := range want { + if files[i] != want[i] { + t.Errorf("files[%d] = %q, want %q", i, files[i], want[i]) + } + } +} + +func TestOrgKnowledgeFilesMissingDir(t *testing.T) { + if _, err := OrgKnowledgeFiles(filepath.Join(t.TempDir(), "missing")); err == nil { + t.Error("OrgKnowledgeFiles() on a missing directory should error") + } +} diff --git a/main.go b/main.go index e0b339f..cd178e4 100644 --- a/main.go +++ b/main.go @@ -187,11 +187,12 @@ func registerTools(s *server.MCPServer, handlerMap map[string]server.ToolHandler // buildHandlers wires the tools package handlers to the resolver/index. func buildHandlers(res *projects.Resolver, idx *search.Index, orgMode bool) map[string]server.ToolHandlerFunc { h := map[string]server.ToolHandlerFunc{ - tools.ToolInitKnowledge: tools.InitHandler(res), - tools.ToolWriteKnowledge: tools.WriteHandler(res, idx), - tools.ToolQueryKnowledge: tools.QueryHandler(res, idx), - tools.ToolListKnowledge: tools.ListHandler(res, idx), - tools.ToolUpdateKnowledge: tools.UpdateHandler(res, idx), + tools.ToolInitKnowledge: tools.InitHandler(res), + tools.ToolWriteKnowledge: tools.WriteHandler(res, idx), + tools.ToolQueryKnowledge: tools.QueryHandler(res, idx), + tools.ToolListKnowledge: tools.ListHandler(res, idx), + tools.ToolUpdateKnowledge: tools.UpdateHandler(res, idx), + tools.ToolReindexKnowledge: tools.ReindexHandler(idx), } if orgMode { h[tools.ToolListProjects] = tools.ListProjectsHandler(res) diff --git a/search/index.go b/search/index.go index 3aef02e..e6f2368 100644 --- a/search/index.go +++ b/search/index.go @@ -1,7 +1,10 @@ package search import ( + "crypto/sha256" + "encoding/hex" "fmt" + "hash" "log" "os" "path/filepath" @@ -37,13 +40,18 @@ type SearchResult struct { } // Index wraps an in-memory bleve index for knowledge search. The index is -// rebuilt from the store files on every startup and lives only as long as -// the process; there is no disk state and therefore no lock between -// instances. +// built from the store files at startup and rebuilt when those files change, +// so out-of-band edits are searchable without a process restart. It lives +// only as long as the process; there is no disk state and therefore no lock +// between instances. type Index struct { - index bleve.Index - ready chan struct{} // closed when indexing completes - readyOnce sync.Once // guards close of ready to prevent double-close panic + mu sync.RWMutex // guards index, fingerprint, and resolver + refreshMu sync.Mutex // serializes rebuilds + index bleve.Index + ready chan struct{} // closed when the initial indexing completes + readyOnce sync.Once // guards close of ready to prevent double-close panic + resolver *projects.Resolver + fingerprint string // store-file stamp the current index was built from } // NewIndex creates a new in-memory bleve index. @@ -55,17 +63,37 @@ func NewIndex() (*Index, error) { return &Index{index: idx, ready: make(chan struct{})}, nil } -// Add indexes a document with the given (scoped) key. +// Add indexes a document with the given (scoped) key. A later Query detects +// that the store files changed and rebuilds; the caller does not have to +// signal anything. func (i *Index) Add(id string, doc SearchDocument) error { + i.mu.RLock() + defer i.mu.RUnlock() + if i.index == nil { + return fmt.Errorf("index is closed") + } return i.index.Index(id, doc) } // Query searches the index, scoped to one project, with full-text -// search and optional filters. +// search and optional filters. When the store files changed since the last +// build (direct edits, git checkout, migration scripts), Query rebuilds the +// index first so results never silently disagree with list_knowledge. func (i *Index) Query(project, q, category string, limit int) ([]SearchResult, error) { - // Block until indexing completes. + // Block until initial indexing completes. <-i.ready + if _, err := i.Refresh(); err != nil { + log.Printf("warning: refresh search index: %v", err) + } + + i.mu.RLock() + defer i.mu.RUnlock() + + if i.index == nil { + return nil, fmt.Errorf("index is closed") + } + if limit <= 0 { limit = 10 } @@ -139,10 +167,17 @@ func (i *Index) Query(project, q, category string, limit int) ([]SearchResult, e // Close closes the bleve index. func (i *Index) Close() error { - if i == nil || i.index == nil { + if i == nil { + return nil + } + i.mu.Lock() + defer i.mu.Unlock() + if i.index == nil { return nil } - return i.index.Close() + err := i.index.Close() + i.index = nil + return err } // CloseReady closes the ready channel exactly once, preventing double-close panics. @@ -158,28 +193,161 @@ func (i *Index) WaitReady() { // DocCount reports the number of documents currently in the index. func (i *Index) DocCount() (uint64, error) { + i.mu.RLock() + defer i.mu.RUnlock() + if i.index == nil { + return 0, fmt.Errorf("index is closed") + } return i.index.DocCount() } +// Refresh rebuilds the index when the store files changed since the last +// build. It reports whether a rebuild happened. The unchanged case takes no +// rebuild lock, so concurrent queries are not serialized by the check. +func (i *Index) Refresh() (bool, error) { + i.mu.RLock() + res := i.resolver + current := i.fingerprint + i.mu.RUnlock() + if res == nil { + return false, nil + } + if storeFingerprint(res) == current { + return false, nil + } + + i.refreshMu.Lock() + defer i.refreshMu.Unlock() + + // Re-check: another goroutine may have rebuilt while we waited. + i.mu.RLock() + res = i.resolver + current = i.fingerprint + i.mu.RUnlock() + if res == nil { + return false, nil + } + fp := storeFingerprint(res) + if fp == current { + return false, nil + } + if err := i.rebuildLocked(res, fp); err != nil { + return false, err + } + return true, nil +} + +// Rebuild re-indexes every known store from disk and swaps the new index in +// atomically. Unlike Refresh it always rebuilds. Queries in flight finish +// against the previous index before it is closed. +func (i *Index) Rebuild() error { + i.refreshMu.Lock() + defer i.refreshMu.Unlock() + + i.mu.RLock() + res := i.resolver + i.mu.RUnlock() + if res == nil { + return fmt.Errorf("index has no resolver; the server has not finished startup indexing") + } + return i.rebuildLocked(res, storeFingerprint(res)) +} + +// rebuildLocked builds a fresh index and swaps it in. Callers must hold +// refreshMu; the old index is closed after the swap. +func (i *Index) rebuildLocked(res *projects.Resolver, fp string) error { + fresh, err := buildIndex(res) + if err != nil { + return err + } + + i.mu.Lock() + old := i.index + i.index = fresh + i.fingerprint = fp + i.mu.Unlock() + + if old != nil { + if closeErr := old.Close(); closeErr != nil { + log.Printf("warning: close replaced search index: %v", closeErr) + } + } + return nil +} + // IndexAll indexes all projects known to the resolver. It closes the ready // channel when complete. func (i *Index) IndexAll(res *projects.Resolver) { - defer i.CloseReady() + i.mu.Lock() + i.resolver = res + i.mu.Unlock() + + i.refreshMu.Lock() + err := i.rebuildLocked(res, storeFingerprint(res)) + i.refreshMu.Unlock() + if err != nil { + log.Printf("warning: build search index: %v", err) + } + + i.CloseReady() +} + +// buildIndex creates a fresh in-memory index from the resolver's stores. +func buildIndex(res *projects.Resolver) (bleve.Index, error) { + idx, err := bleve.NewMemOnly(indexMapping()) + if err != nil { + return nil, fmt.Errorf("create in-memory index: %w", err) + } for _, ref := range res.Snapshot() { switch ref.Kind { - case projects.KindProject: - i.indexProjectKnowledge(res.AgentsDir(ref), ref.Address) case projects.KindOrg: - i.indexOrgKnowledge(res.OrgKnowledgeDir(ref), ref.Name) - case projects.KindGlobal: - i.indexProjectKnowledge(res.AgentsDir(ref), ref.Address) + indexOrgKnowledge(idx, res.OrgKnowledgeDir(ref), ref.Name) + default: + indexProjectKnowledge(idx, res.AgentsDir(ref), ref.Address) + } + } + return idx, nil +} + +// storeFingerprint hashes the identity of every knowledge source file the +// index is built from: path, size, and modification time. Missing files are +// part of the stamp so creation and deletion both trigger a rebuild. +func storeFingerprint(res *projects.Resolver) string { + sum := sha256.New() + for _, ref := range res.Snapshot() { + fmt.Fprintf(sum, "%s:%d\n", ref.Address, ref.Kind) + if ref.Kind == projects.KindOrg { + dir := res.OrgKnowledgeDir(ref) + files, err := knowledge.OrgKnowledgeFiles(dir) + if err != nil { + fmt.Fprintf(sum, "error:%v\n", err) + continue + } + for _, name := range files { + hashFileStamp(sum, filepath.Join(dir, name)) + } + continue + } + for _, cat := range knowledge.ValidCategories() { + hashFileStamp(sum, knowledge.CategoryFilePath(res.AgentsDir(ref), cat)) } } + return hex.EncodeToString(sum.Sum(nil)) +} + +// hashFileStamp writes path/size/mtime (or a missing marker) into h. +func hashFileStamp(h hash.Hash, path string) { + info, err := os.Stat(path) + if err != nil { + fmt.Fprintf(h, "%s:missing\n", path) + return + } + fmt.Fprintf(h, "%s:%d:%d\n", path, info.Size(), info.ModTime().UnixNano()) } // indexProjectKnowledge indexes all knowledge files under an agents dir. -func (i *Index) indexProjectKnowledge(agentsDir, projectName string) { +func indexProjectKnowledge(idx bleve.Index, agentsDir, projectName string) { for _, cat := range knowledge.ValidCategories() { catPath := knowledge.CategoryFilePath(agentsDir, cat) kf, err := knowledge.Load(catPath) @@ -197,19 +365,26 @@ func (i *Index) indexProjectKnowledge(agentsDir, projectName string) { Category: cat, Project: projectName, } - if addErr := i.Add(projectName+"/"+entry.ID, doc); addErr != nil { + if addErr := idx.Index(projectName+"/"+entry.ID, doc); addErr != nil { log.Printf("warning: failed to index %s/%s: %v", projectName, entry.ID, addErr) } } } } -// indexOrgKnowledge indexes an org's knowledge files as sections. -func (i *Index) indexOrgKnowledge(knowledgeDir, orgName string) { - for _, catFile := range []string{"architecture.md", "review.md"} { - filePath := filepath.Join(knowledgeDir, catFile) - data, err := os.ReadFile(filePath) - if err != nil { +// indexOrgKnowledge indexes every markdown document in an org's knowledge +// dir as sections. +func indexOrgKnowledge(idx bleve.Index, knowledgeDir, orgName string) { + files, err := knowledge.OrgKnowledgeFiles(knowledgeDir) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("warning: failed to list org knowledge %s: %v", knowledgeDir, err) + } + return + } + for _, catFile := range files { + data, readErr := os.ReadFile(filepath.Join(knowledgeDir, catFile)) + if readErr != nil { continue } for _, sec := range knowledge.SplitSections(string(data)) { @@ -221,7 +396,7 @@ func (i *Index) indexOrgKnowledge(knowledgeDir, orgName string) { Project: orgName, } id := fmt.Sprintf("%s/org-%s::%s", orgName, catFile, heading) - if addErr := i.Add(id, doc); addErr != nil { + if addErr := idx.Index(id, doc); addErr != nil { log.Printf("warning: failed to index %s: %v", id, addErr) } } diff --git a/search/refresh_test.go b/search/refresh_test.go new file mode 100644 index 0000000..89562e2 --- /dev/null +++ b/search/refresh_test.go @@ -0,0 +1,254 @@ +package search + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/renderorange/knowledge-mcp/knowledge" + "github.com/renderorange/knowledge-mcp/projects" +) + +// saveEntries writes a category file directly, bypassing Index.Add. This is +// the out-of-band edit (git checkout, migration script, manual edit) that +// query_knowledge must notice. +func saveEntries(t *testing.T, agentsDir, category string, entries ...knowledge.Entry) { + t.Helper() + if err := knowledge.EnsureDir(agentsDir); err != nil { + t.Fatalf("EnsureDir() error: %v", err) + } + kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: entries} + if err := knowledge.Save(knowledge.CategoryFilePath(agentsDir, category), kf); err != nil { + t.Fatalf("knowledge.Save() error: %v", err) + } +} + +func testEntry(id, summary, detail string) knowledge.Entry { + return knowledge.Entry{ + ID: id, Summary: summary, Detail: detail, Source: "test", Date: "2026-10-23", + } +} + +func buildProjectIndex(t *testing.T, proj string) *Index { + t.Helper() + resolver, _, err := projects.BuildWithStore(nil, []string{proj}, "", "") + if err != nil { + t.Fatalf("BuildWithStore() error: %v", err) + } + idx := newIndex(t) + idx.IndexAll(resolver) + return idx +} + +func TestQueryRefreshesAfterDirectEdit(t *testing.T) { + proj := filepath.Join(t.TempDir(), "refreshproj") + agents := filepath.Join(proj, ".agents") + first := testEntry("sub-001", "Storage backend", "RustFS keeps objects") + saveEntries(t, agents, "subsystems", first) + + idx := buildProjectIndex(t, proj) + defer idx.Close() + + results, err := idx.Query("refreshproj", "RustFS", "", 10) + if err != nil { + t.Fatalf("Query() error: %v", err) + } + if len(results) != 1 { + t.Fatalf("len(results) = %d, want 1", len(results)) + } + + // Direct edit: add a second entry without calling Index.Add. + second := testEntry("sub-002", "Trash sweep", "SweepTrash purges stale rows") + saveEntries(t, agents, "subsystems", first, second) + + results, err = idx.Query("refreshproj", "SweepTrash", "", 10) + if err != nil { + t.Fatalf("Query() after direct edit error: %v", err) + } + if len(results) != 1 { + t.Fatalf("len(results) after direct edit = %d, want 1", len(results)) + } + if results[0].ID != "sub-002" { + t.Errorf("ID = %q, want %q", results[0].ID, "sub-002") + } +} + +func TestQueryDropsRemovedEntry(t *testing.T) { + proj := filepath.Join(t.TempDir(), "deleteproj") + agents := filepath.Join(proj, ".agents") + keep := testEntry("sub-001", "Storage backend", "RustFS keeps objects") + remove := testEntry("sub-002", "Trash sweep", "SweepTrash purges stale rows") + saveEntries(t, agents, "subsystems", keep, remove) + + idx := buildProjectIndex(t, proj) + defer idx.Close() + + results, err := idx.Query("deleteproj", "SweepTrash", "", 10) + if err != nil { + t.Fatalf("Query() error: %v", err) + } + if len(results) != 1 { + t.Fatalf("len(results) = %d, want 1", len(results)) + } + + // Delete the entry by rewriting the file directly. + saveEntries(t, agents, "subsystems", keep) + + results, err = idx.Query("deleteproj", "SweepTrash", "", 10) + if err != nil { + t.Fatalf("Query() after delete error: %v", err) + } + if len(results) != 0 { + t.Fatalf("len(results) after delete = %d, want 0; stale doc still indexed: %+v", len(results), results) + } +} + +func TestRefreshReportsWhetherStoreChanged(t *testing.T) { + proj := filepath.Join(t.TempDir(), "refreshflag") + agents := filepath.Join(proj, ".agents") + first := testEntry("sub-001", "Storage backend", "RustFS keeps objects") + saveEntries(t, agents, "subsystems", first) + + idx := buildProjectIndex(t, proj) + defer idx.Close() + + changed, err := idx.Refresh() + if err != nil { + t.Fatalf("Refresh() error: %v", err) + } + if changed { + t.Error("Refresh() reported a change for an unchanged store") + } + + saveEntries(t, agents, "subsystems", first, testEntry("sub-002", "Trash sweep", "SweepTrash")) + + changed, err = idx.Refresh() + if err != nil { + t.Fatalf("Refresh() error: %v", err) + } + if !changed { + t.Error("Refresh() did not report a change after a direct edit") + } + + changed, err = idx.Refresh() + if err != nil { + t.Fatalf("Refresh() error: %v", err) + } + if changed { + t.Error("Refresh() reported a second change with no intervening edit") + } +} + +func TestRebuildRefreshesIndex(t *testing.T) { + proj := filepath.Join(t.TempDir(), "rebuildproj") + agents := filepath.Join(proj, ".agents") + first := testEntry("sub-001", "Storage backend", "RustFS keeps objects") + saveEntries(t, agents, "subsystems", first) + + idx := buildProjectIndex(t, proj) + defer idx.Close() + + saveEntries(t, agents, "subsystems", first, testEntry("sub-002", "Trash sweep", "SweepTrash")) + + if err := idx.Rebuild(); err != nil { + t.Fatalf("Rebuild() error: %v", err) + } + + count, err := idx.DocCount() + if err != nil { + t.Fatalf("DocCount() error: %v", err) + } + if count != 2 { + t.Errorf("DocCount() = %d, want 2", count) + } +} + +func TestQueryConcurrentWithRefresh(t *testing.T) { + proj := filepath.Join(t.TempDir(), "raceproj") + agents := filepath.Join(proj, ".agents") + base := testEntry("sub-001", "Storage backend", "RustFS keeps objects") + saveEntries(t, agents, "subsystems", base) + + idx := buildProjectIndex(t, proj) + defer idx.Close() + + var wg sync.WaitGroup + errs := make(chan error, 4) + for g := 0; g < 4; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for n := 0; n < 25; n++ { + if _, err := idx.Query("raceproj", "RustFS", "", 10); err != nil { + errs <- err + return + } + } + }() + } + + entries := []knowledge.Entry{base} + for n := 0; n < 10; n++ { + entries = append(entries, testEntry(fmt.Sprintf("sub-%03d", n+2), fmt.Sprintf("Filler %d", n), "filler detail")) + saveEntries(t, agents, "subsystems", entries...) + if _, err := idx.Refresh(); err != nil { + t.Fatalf("Refresh() error: %v", err) + } + } + + wg.Wait() + close(errs) + for err := range errs { + t.Errorf("concurrent Query() error: %v", err) + } +} + +func TestIndexOrgKnowledgeIndexesAllMarkdown(t *testing.T) { + root := t.TempDir() + knowledgeDir := filepath.Join(root, ".agents", "knowledge") + if err := knowledge.EnsureDir(knowledgeDir); err != nil { + t.Fatalf("EnsureDir() error: %v", err) + } + files := map[string]string{ + "architecture.md": "## Build\nmake build\n", + "review.md": "## Overview\nchaon-review\n", + "notes.md": "## Takedown\npurge_requested_at drives takedown\n", + "ignored.txt": "## Not Markdown\nSweepTrash\n", + } + for name, body := range files { + if err := os.WriteFile(filepath.Join(knowledgeDir, name), []byte(body), 0644); err != nil { + t.Fatalf("WriteFile(%s) error: %v", name, err) + } + } + + resolver, _, err := projects.Build([]string{root}, nil, "") + if err != nil { + t.Fatalf("Build() error: %v", err) + } + rootName := filepath.Base(root) + + idx := newIndex(t) + defer idx.Close() + idx.IndexAll(resolver) + + results, err := idx.Query(rootName, "takedown", "", 10) + if err != nil { + t.Fatalf("Query() error: %v", err) + } + if len(results) != 1 { + t.Fatalf("len(results) = %d, want 1 (notes.md section)", len(results)) + } + if want := "org-notes.md::Takedown"; results[0].ID != want { + t.Errorf("ID = %q, want %q", results[0].ID, want) + } + + results, err = idx.Query(rootName, "SweepTrash", "", 10) + if err != nil { + t.Fatalf("Query() error: %v", err) + } + if len(results) != 0 { + t.Errorf("len(results) = %d for non-markdown file, want 0", len(results)) + } +} diff --git a/tools/registry.go b/tools/registry.go index 5946d92..ab1e3b5 100644 --- a/tools/registry.go +++ b/tools/registry.go @@ -16,12 +16,13 @@ func categoryNames() string { // Tool name constants keep main.go, the registry, and generated // instructions in sync. const ( - ToolInitKnowledge = "init_knowledge" - ToolWriteKnowledge = "write_knowledge" - ToolQueryKnowledge = "query_knowledge" - ToolListKnowledge = "list_knowledge" - ToolUpdateKnowledge = "update_knowledge" - ToolListProjects = "list_projects" + ToolInitKnowledge = "init_knowledge" + ToolWriteKnowledge = "write_knowledge" + ToolQueryKnowledge = "query_knowledge" + ToolListKnowledge = "list_knowledge" + ToolUpdateKnowledge = "update_knowledge" + ToolListProjects = "list_projects" + ToolReindexKnowledge = "reindex_knowledge" ) // ParamSpec describes one tool parameter. @@ -96,6 +97,10 @@ var Registry = []ToolSpec{ Name: ToolListProjects, Purpose: "List all discovered projects (org-wide mode)", Description: "List all discovered projects across configured roots", }, + { + Name: ToolReindexKnowledge, Purpose: "Rebuild the search index from store files", + Description: "Rebuild the in-memory search index from the knowledge store files. Queries refresh automatically when store files change; use this to force a full rebuild.", + }, } // MCPTool converts the spec into an mcp.Tool for server registration. diff --git a/tools/registry_test.go b/tools/registry_test.go index ad8ed41..7e4288e 100644 --- a/tools/registry_test.go +++ b/tools/registry_test.go @@ -6,6 +6,7 @@ func TestRegistryComplete(t *testing.T) { want := map[string]bool{ "init_knowledge": true, "write_knowledge": true, "query_knowledge": true, "list_knowledge": true, "update_knowledge": true, "list_projects": true, + "reindex_knowledge": true, } seen := map[string]bool{} for _, spec := range Registry { diff --git a/tools/reindex.go b/tools/reindex.go new file mode 100644 index 0000000..563a322 --- /dev/null +++ b/tools/reindex.go @@ -0,0 +1,28 @@ +package tools + +import ( + "context" + "fmt" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/renderorange/knowledge-mcp/search" +) + +// ReindexHandler handles the reindex_knowledge MCP tool. Queries already +// refresh automatically when store files change; this tool forces a full +// rebuild on demand. +func ReindexHandler(idx *search.Index) func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if idx == nil { + return mcp.NewToolResultError("search index is not available"), nil + } + if err := idx.Rebuild(); err != nil { + return mcp.NewToolResultError(fmt.Sprintf("reindex failed: %v", err)), nil + } + count, err := idx.DocCount() + if err != nil { + return mcp.NewToolResultError(fmt.Sprintf("reindexed but could not count documents: %v", err)), nil + } + return mcp.NewToolResultText(fmt.Sprintf("reindexed %d documents from the knowledge store", count)), nil + } +} diff --git a/tools/reindex_test.go b/tools/reindex_test.go new file mode 100644 index 0000000..e11f315 --- /dev/null +++ b/tools/reindex_test.go @@ -0,0 +1,122 @@ +package tools + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/renderorange/knowledge-mcp/knowledge" + "github.com/renderorange/knowledge-mcp/projects" +) + +func TestReindexHandlerRebuildsIndex(t *testing.T) { + proj := filepath.Join(t.TempDir(), "reindexproj") + agents := filepath.Join(proj, ".agents") + if err := knowledge.EnsureDir(agents); err != nil { + t.Fatalf("EnsureDir() error: %v", err) + } + kf := &knowledge.KnowledgeFile{Project: "reindexproj", Version: 1, Entries: []knowledge.Entry{ + {ID: "sub-001", Summary: "Storage backend", Detail: "RustFS keeps objects", Source: "test", Date: "2026-10-23"}, + {ID: "sub-002", Summary: "Trash sweep", Detail: "SweepTrash purges rows", Source: "test", Date: "2026-10-23"}, + }} + if err := knowledge.Save(knowledge.CategoryFilePath(agents, "subsystems"), kf); err != nil { + t.Fatalf("knowledge.Save() error: %v", err) + } + + resolver, _, err := projects.BuildWithStore(nil, []string{proj}, "", "") + if err != nil { + t.Fatalf("BuildWithStore() error: %v", err) + } + + idx := newIndexReady(t) + defer idx.Close() + idx.IndexAll(resolver) + + handler := ReindexHandler(idx) + result, err := handler(context.Background(), mcp.CallToolRequest{}) + if err != nil { + t.Fatalf("ReindexHandler() error: %v", err) + } + if result.IsError { + t.Fatalf("ReindexHandler() returned error result: %s", extractTextContent(t, result)) + } + + content := extractTextContent(t, result) + if !strings.Contains(content, "2 documents") { + t.Errorf("content = %q, want it to report 2 documents", content) + } +} + +func TestReindexHandlerWithoutResolver(t *testing.T) { + idx := newIndexReady(t) + defer idx.Close() + + result, err := ReindexHandler(idx)(context.Background(), mcp.CallToolRequest{}) + if err != nil { + t.Fatalf("ReindexHandler() error: %v", err) + } + if !result.IsError { + t.Error("ReindexHandler() without a resolver should return an error result") + } +} + +func TestReindexHandlerNilIndex(t *testing.T) { + result, err := ReindexHandler(nil)(context.Background(), mcp.CallToolRequest{}) + if err != nil { + t.Fatalf("ReindexHandler() error: %v", err) + } + if !result.IsError { + t.Error("ReindexHandler(nil) should return an error result") + } +} + +// TestWriteThenQueryOnLiveIndex covers the server flow: the index was built +// from a resolver, write_knowledge saves the file and Adds the doc, and the +// next query must see the entry even though the store fingerprint changed. +func TestWriteThenQueryOnLiveIndex(t *testing.T) { + proj := filepath.Join(t.TempDir(), "liveproj") + if err := knowledge.EnsureDir(filepath.Join(proj, ".agents")); err != nil { + t.Fatalf("EnsureDir() error: %v", err) + } + + resolver, _, err := projects.BuildWithStore(nil, []string{proj}, "", "") + if err != nil { + t.Fatalf("BuildWithStore() error: %v", err) + } + + idx := newIndexReady(t) + defer idx.Close() + idx.IndexAll(resolver) + + writeReq := mcp.CallToolRequest{} + writeReq.Params.Arguments = map[string]interface{}{ + "project": "liveproj", + "category": "subsystems", + "summary": "Trash sweep", + "detail": "SweepTrash purges stale rows", + "source": "test", + } + writeResult, err := WriteHandler(resolver, idx)(context.Background(), writeReq) + if err != nil { + t.Fatalf("WriteHandler() error: %v", err) + } + if writeResult.IsError { + t.Fatalf("WriteHandler() error result: %s", extractTextContent(t, writeResult)) + } + + queryReq := mcp.CallToolRequest{} + queryReq.Params.Arguments = map[string]interface{}{ + "project": "liveproj", + "query": "SweepTrash", + } + queryResult, err := QueryHandler(resolver, idx)(context.Background(), queryReq) + if err != nil { + t.Fatalf("QueryHandler() error: %v", err) + } + content := extractTextContent(t, queryResult) + if !strings.Contains(content, "sub-001") { + t.Errorf("query after write did not find the entry: %s", content) + } +}