GH-36: refresh the index when store files change

.agents/decisions.yaml

@@ -2,7 +2,7 @@ project: knowledge-mcp
 version: 1
 entries:
     - id: dec-001
-      summary: "Org-level knowledge is indexed per markdown section, not as whole files"
+      summary: Org-level knowledge is indexed per markdown section, not as whole files
       detail: |-
         Whole-file indexing made org queries return 5-6K tokens of unrelated content (measured 2026-09-08: "daisy test-stability" on chaon-io returned full architecture.md and review.md).
         Split .agents/knowledge/*.md on "## " headings; each section is its own document with key org-<file>::<heading>.
@@ -11,27 +11,40 @@ entries:
       source: user-requested token-savings fix, 2026-09-08; GH-34 update, 2026-10-07
       date: "2026-10-07"
     - id: dec-002
-      summary: "Merged global entries carry explicit (global) provenance in list output"
+      summary: Merged global entries carry explicit (global) provenance in list output
       detail: |-
         list_knowledge for any non-global project prepends _global entries; previously they were indistinguishable from the project's own entries.
         Section headers now carry a " (global)" suffix; projects with no local entries get an explicit "no project-specific knowledge entries found" note.
         Query results already carried a project field, so only the list presentation changed.
       source: user-requested provenance fix, 2026-09-08
       date: "2026-09-08"
     - id: dec-003
-      summary: "Query text is escaped so special characters are always literal"
+      summary: Query text is escaped so special characters are always literal
       detail: |-
         User queries are plain text, not lucene syntax; characters like : ( ) && || " were previously interpreted as operators.
         escapeQueryString escapes syntax-active characters (context-aware for +/- and &&/||); mid-word hyphens like test-stability stay intact.
         Decision documented so future search work does not re-introduce raw QueryStringQuery usage.
       source: user-requested robustness fix, 2026-09-08
       date: "2026-09-08"
     - id: dec-004
-      summary: "Startup indexing: always built in memory (no skip flag)"
+      summary: 'Startup indexing: always built in memory (no skip flag)'
       detail: |-
         The former background-indexing flag --no-index-on-startup is now a deprecated no-op (accepted, warns, ignored).
         The index is always built in memory at startup; queries block on the ready channel until IndexAll completes.
         readyOnce (sync.Once) prevents double-close panics.
         Recorded so a skip-indexing flag is not reintroduced.
       source: GH-34 memory-only index, 2026-10-07
-      date: "2026-10-07"
+      date: "2026-10-07"
+    - id: dec-005
+      summary: Stale index fixed by fingerprint auto-refresh plus a forced reindex tool
+      detail: |-
+        Issue #36: the index was built once at startup, so direct store edits were visible to list_knowledge but not query_knowledge, silently.
+        Chosen fix (user-approved): build-then-swap on staleness, plus an explicit reindex_knowledge tool.
+        Why not re-run IndexAll as-is: Add is an upsert keyed by project/id and bleve has no Delete wired up, so entries removed from the files would keep matching. A fresh index per rebuild drops them naturally.
+        Fingerprint = sha256 over each ref's address/kind and each source file's path, size and mtime; missing files are part of the stamp so creation and deletion both trigger a rebuild.
+        Query computes the fingerprint before searching and rebuilds under refreshMu; the swap takes the write lock so in-flight queries finish on the old index before it is closed.
+        Add does not update the fingerprint: the next query rebuilds after a tool write, which is slightly wasteful but cannot swallow a concurrent out-of-band change.
+        Org knowledge now indexes every top-level *.md in .agents/knowledge/ (shared helper knowledge.OrgKnowledgeFiles) instead of the architecture.md/review.md allowlist, in both the index and the Grep/Glob hook.
+        list_knowledge stays file-backed for projects and index-backed for orgs; with auto-refresh both are current.
+      source: 'issue #36 design discussion with user, 2026-10-23'
+      date: "2026-10-23"

.agents/subsystems.yaml

@@ -2,19 +2,21 @@ project: knowledge-mcp
 version: 1
 entries:
     - id: sub-001
-      summary: "Search index: per-process in-memory bleve, scoped keys, org sections"
+      summary: 'Search index: in-memory bleve, auto-refresh on store edits, scoped keys'
       detail: |-
         The index is built in memory (bleve.NewMemOnly) at server startup from the store YAML/MD files — every restart re-indexes.
         There is no disk state, no index lock, and any number of instances may query concurrently.
+        A store fingerprint (per-file path, size, mtime) is recorded at build time; Query recomputes it and rebuilds into a fresh index when the files changed, so direct edits, git checkouts and migration scripts are searchable without a restart.
+        Rebuild is build-then-swap under an RWMutex: queries in flight finish on the old index, then it is closed. There is no Delete API, so deletes vanish only because the replacement index is built from scratch.
+        reindex_knowledge forces a rebuild; Refresh reports whether the fingerprint changed.
         Documents carry summary, detail, rule, category, project with scoped keys <project>/<entryID>.
-        Query results de-scope the ID.
-        Org-level docs (.agents/knowledge/*.md) are split on "## " headings with keys org-<file>::<heading>.
+        Query results de-scope the ID. Org-level docs: every top-level *.md in .agents/knowledge/ (knowledge.OrgKnowledgeFiles), split on "## " headings with keys org-<file>::<heading>.
         User query text is escaped (escapeQueryString) so special characters are literal, never lucene syntax.
         --index and --no-index-on-startup are accepted warning-logged no-ops.
       source: GH-34 memory-only index, 2026-10-07
-      date: "2026-10-07"
+      date: "2026-10-23"
     - id: sub-002
-      summary: "Project resolver: kinds, addressing, dynamic fallback"
+      summary: 'Project resolver: kinds, addressing, dynamic fallback'
       detail: |-
         projects/resolver.go builds refs: KindProject (has .agents/ yaml), KindOrg (root with .agents/knowledge/), KindGlobal (--global path, addressed _global).
         Roots scan immediate children only, skipping .agents and .git; symlinks followed; unreadable entries warn.
@@ -23,7 +25,7 @@ entries:
       source: code review, 2026-09-08; GH-34 update, 2026-10-07
       date: "2026-10-07"
     - id: sub-003
-      summary: "Knowledge store: atomic writes, per-file locks, entry lifecycle"
+      summary: 'Knowledge store: atomic writes, per-file locks, entry lifecycle'
       detail: |-
         knowledge/store.go: Save writes temp file, fsyncs, renames (atomic); LoadOrCreate returns empty KnowledgeFile when missing.
         tools package keeps per-path RWMutexes (fileLocks) around read-modify-write cycles; write/update lock, list takes RLock.
@@ -34,12 +36,12 @@ entries:
       source: code review, 2026-09-08
       date: "2026-09-08"
     - id: sub-004
-      summary: "Startup indexing: always-on background kick, no skip flag"
+      summary: 'Startup indexing: always-on background kick, no skip flag'
       detail: |-
         IndexAll always runs in a background goroutine at startup (there is no skip flag).
         Queries block on the ready channel until IndexAll completes.
         readyOnce (sync.Once) prevents a double-close panic on the ready channel.
         --no-index-on-startup and --index are deprecated, accepted, warn, and do nothing.
         The index lives only as long as the process and is rebuilt from the store files on every startup.
       source: GH-34 memory-only index, 2026-10-07
-      date: "2026-10-07"
+      date: "2026-10-07"

README.md

@@ -177,10 +177,14 @@ General notes:
 
 - Each `--root` discovers its immediate children as projects (one level deep — pass deeper directories as additional flags).
 - Duplicate project basenames across roots are addressed as `<root>/<project>` (e.g. `work/api`); `list_projects` shows which names need qualification.
-- `query_knowledge` accepts org root names to search that root's `.agents/knowledge/` files. Org-level documents are indexed **per `##` section**, so a query returns the matching section(s), not the entire file.
+- `query_knowledge` accepts org root names to search that root's `.agents/knowledge/` files. Every `.md` file in that directory is indexed (not a fixed filename allowlist), and documents are indexed **per `##` section**, so a query returns the matching section(s), not the entire file.
 - The search index is built **in memory** per server process at startup and
   does not survive a restart. There is no index on disk anywhere, no lock,
   and no contention — any number of instances may query concurrently.
+- The index notices out-of-band store edits (hand edits, `git checkout`,
+  migration scripts) and rebuilds itself before serving the next query, so
+  `query_knowledge` never silently disagrees with `list_knowledge`. Use
+  `reindex_knowledge` to force a full rebuild.
 - `--index <path>` is deprecated and ignored (the search index is in-memory).
 - Old leftover `.agents/.index/` and `<store>/.index/` directories and `$XDG_STATE_HOME/knowledge-mcp/` artifacts from earlier releases are inert and can be deleted.
 
@@ -222,6 +226,7 @@ arguments — entry bodies are replaced by sizes and never written. Set
 | `list_knowledge` | List all entries; entries with rules shown first as constraints |
 | `update_knowledge` | Update an existing entry by ID |
 | `list_projects` | List all discovered projects (org-wide mode only) |
+| `reindex_knowledge` | Rebuild the search index from the store files (queries also refresh automatically) |
 
 Queries are plain-text search with no operator syntax — special characters in queries are always treated as literal text.
 

hook/match_test.go

@@ -105,6 +105,29 @@ func TestSearchIncludesOrgAndGlobal(t *testing.T) {
 	}
 }
 
+// TestSearchIncludesAdditionalOrgMarkdown covers the org knowledge glob:
+// every markdown file in .agents/knowledge/ is searched, not just the
+// historical architecture.md/review.md pair.
+func TestSearchIncludesAdditionalOrgMarkdown(t *testing.T) {
+	root := t.TempDir()
+	orgKnowledge := filepath.Join(root, ".agents", "knowledge")
+	if err := os.MkdirAll(orgKnowledge, 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(orgKnowledge, "notes.md"),
+		[]byte("## Takedown\npurge_requested_at drives takedown\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	res, _, err := projects.Build([]string{root}, nil, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	hits := Search(res, nil, orgRef(t, res), "takedown", 3, nil)
+	if len(hits) != 1 || hits[0].ID != "org-notes.md::Takedown" {
+		t.Fatalf("hits = %+v", hits)
+	}
+}
+
 func orgRef(t *testing.T, res *projects.Resolver) *projects.Ref {
 	t.Helper()
 	for _, ref := range res.Snapshot() {

hook/search.go

@@ -45,14 +45,17 @@ func Search(res *projects.Resolver, proj, org *projects.Ref, pattern string, lim
 		}
 	}
 	if org != nil {
-		for _, file := range []string{"architecture.md", "review.md"} {
-			data, err := os.ReadFile(filepath.Join(res.OrgKnowledgeDir(*org), file))
-			if err != nil {
-				continue
-			}
-			files++
-			for _, sec := range knowledge.SplitSections(string(data)) {
-				addEntry(org.Address, "org-"+file+"::"+sec[0], file+": "+sec[0], sec[1], "")
+		names, err := knowledge.OrgKnowledgeFiles(res.OrgKnowledgeDir(*org))
+		if err == nil {
+			for _, file := range names {
+				data, readErr := os.ReadFile(filepath.Join(res.OrgKnowledgeDir(*org), file))
+				if readErr != nil {
+					continue
+				}
+				files++
+				for _, sec := range knowledge.SplitSections(string(data)) {
+					addEntry(org.Address, "org-"+file+"::"+sec[0], file+": "+sec[0], sec[1], "")
+				}
 			}
 		}
 	}

install/skill.go

@@ -14,7 +14,7 @@ func RenderSkillMD(registry []tools.ToolSpec) []byte {
 	}
 	md := "---\n" +
 		"name: knowledge-mcp\n" +
-		"description: \"Use the persistent knowledge store for project conventions, subsystems, decisions, and workflows. Triggers on: what are our conventions, does this project have rules about, how does this subsystem work, why was this decision made, run the brainstorm workflow, run the plan workflow, debug workflow, verify workflow, capture this decision, write that down as knowledge, update that entry, knowledge store, .agents, list_knowledge, query_knowledge, write_knowledge, update_knowledge, init_knowledge.\"\n" +
+		"description: \"Use the persistent knowledge store for project conventions, subsystems, decisions, and workflows. Triggers on: what are our conventions, does this project have rules about, how does this subsystem work, why was this decision made, run the brainstorm workflow, run the plan workflow, debug workflow, verify workflow, capture this decision, write that down as knowledge, update that entry, knowledge store, .agents, list_knowledge, query_knowledge, write_knowledge, update_knowledge, init_knowledge, reindex_knowledge.\"\n" +
 		"---\n" +
 		"\n" +
 		"# Knowledge Store (knowledge-mcp)\n" +

knowledge/store.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -118,6 +119,24 @@ func LoadOrCreate(filePath, project string) (*KnowledgeFile, error) {
 	return kf, nil
 }
 
+// OrgKnowledgeFiles returns the sorted names of the markdown files directly
+// under an org knowledge directory. Subdirectories are not walked.
+func OrgKnowledgeFiles(dir string) ([]string, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	var files []string
+	for _, entry := range entries {
+		if entry.IsDir() || !strings.EqualFold(filepath.Ext(entry.Name()), ".md") {
+			continue
+		}
+		files = append(files, entry.Name())
+	}
+	sort.Strings(files)
+	return files, nil
+}
+
 // SplitSections splits a knowledge markdown document on lines starting
 // with "## ". Content before the first section heading is kept under
 // "Overview". Subsection headings (### ...) stay part of their parent

knowledge/store_test.go

@@ -170,3 +170,38 @@ func TestEnsureDir(t *testing.T) {
 		t.Fatalf("EnsureDir() second call error: %v", err)
 	}
 }
+
+func TestOrgKnowledgeFiles(t *testing.T) {
+	dir := t.TempDir()
+	for _, name := range []string{"review.md", "architecture.md", "notes.md", "ignored.txt"} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte("## Section\nbody\n"), 0644); err != nil {
+			t.Fatalf("WriteFile(%s) error: %v", name, err)
+		}
+	}
+	if err := os.Mkdir(filepath.Join(dir, "nested"), 0755); err != nil {
+		t.Fatalf("Mkdir() error: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "nested", "deep.md"), []byte("## Deep\nbody\n"), 0644); err != nil {
+		t.Fatalf("WriteFile(nested) error: %v", err)
+	}
+
+	files, err := OrgKnowledgeFiles(dir)
+	if err != nil {
+		t.Fatalf("OrgKnowledgeFiles() error: %v", err)
+	}
+	want := []string{"architecture.md", "notes.md", "review.md"}
+	if len(files) != len(want) {
+		t.Fatalf("OrgKnowledgeFiles() = %v, want %v", files, want)
+	}
+	for i := range want {
+		if files[i] != want[i] {
+			t.Errorf("files[%d] = %q, want %q", i, files[i], want[i])
+		}
+	}
+}
+
+func TestOrgKnowledgeFilesMissingDir(t *testing.T) {
+	if _, err := OrgKnowledgeFiles(filepath.Join(t.TempDir(), "missing")); err == nil {
+		t.Error("OrgKnowledgeFiles() on a missing directory should error")
+	}
+}

main.go

@@ -187,11 +187,12 @@ func registerTools(s *server.MCPServer, handlerMap map[string]server.ToolHandler
 // buildHandlers wires the tools package handlers to the resolver/index.
 func buildHandlers(res *projects.Resolver, idx *search.Index, orgMode bool) map[string]server.ToolHandlerFunc {
 	h := map[string]server.ToolHandlerFunc{
-		tools.ToolInitKnowledge:   tools.InitHandler(res),
-		tools.ToolWriteKnowledge:  tools.WriteHandler(res, idx),
-		tools.ToolQueryKnowledge:  tools.QueryHandler(res, idx),
-		tools.ToolListKnowledge:   tools.ListHandler(res, idx),
-		tools.ToolUpdateKnowledge: tools.UpdateHandler(res, idx),
+		tools.ToolInitKnowledge:    tools.InitHandler(res),
+		tools.ToolWriteKnowledge:   tools.WriteHandler(res, idx),
+		tools.ToolQueryKnowledge:   tools.QueryHandler(res, idx),
+		tools.ToolListKnowledge:    tools.ListHandler(res, idx),
+		tools.ToolUpdateKnowledge:  tools.UpdateHandler(res, idx),
+		tools.ToolReindexKnowledge: tools.ReindexHandler(idx),
 	}
 	if orgMode {
 		h[tools.ToolListProjects] = tools.ListProjectsHandler(res)