From d7ca4f0b0f8830b73caaf1fbfeaff29c9ff6d734 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 10:34:53 -0400 Subject: [PATCH 1/6] GH-34: replace disk index with per-process memory index --- search/index.go | 316 +++++----------------------------------- search/index_test.go | 279 +++++------------------------------ search/loadwarn_test.go | 2 +- search/location.go | 39 ----- search/location_test.go | 115 --------------- search/race_test.go | 125 ---------------- 6 files changed, 71 insertions(+), 805 deletions(-) delete mode 100644 search/location.go delete mode 100644 search/location_test.go delete mode 100644 search/race_test.go diff --git a/search/index.go b/search/index.go index e7c1c4e..3aef02e 100644 --- a/search/index.go +++ b/search/index.go @@ -1,17 +1,12 @@ package search import ( - "encoding/json" - "errors" "fmt" "log" "os" "path/filepath" - "slices" - "sort" "strings" "sync" - "time" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/mapping" @@ -20,20 +15,6 @@ import ( "github.com/renderorange/knowledge-mcp/projects" ) -// metaFileVersion is the on-disk key-format version. A mismatch forces an index rebuild. -// Version 3: org-level knowledge docs are indexed per markdown section -// instead of as whole files. -const metaFileVersion = 3 - -// indexMeta is stored in a sibling file next to the bleve index directory. -// It records the key format and the project-name set the index was built with; -// a change in either means the on-disk keys no longer match, so the index is -// wiped and rebuilt from the yaml source of truth. -type indexMeta struct { - Version int `json:"version"` - Names []string `json:"names"` -} - // SearchDocument is the document indexed by bleve. type SearchDocument struct { Summary string `json:"summary"` @@ -55,264 +36,34 @@ type SearchResult struct { Project string `json:"project"` } -// Index wraps a bleve index for knowledge search. +// Index wraps an in-memory bleve index for knowledge search. The index is +// rebuilt from the store files on every startup and lives only as long as +// the process; there is no disk state and therefore no lock between +// instances. type Index struct { - indexPath string - names []string index bleve.Index - ready chan struct{} // closed when indexing completes or stale data exists + ready chan struct{} // closed when indexing completes readyOnce sync.Once // guards close of ready to prevent double-close panic - - openMu sync.Mutex - openStarted bool // guards against double opens - openDone chan struct{} // closed when the open attempt resolves - openErr error // set when the open attempt fails; read after openDone closes -} - -// openWaitTimeout bounds how long Query/Add wait on a pending index open. -// The only expected long-open case is another instance holding the -// single-writer bbolt lock; everything else opens in milliseconds. -const openWaitTimeout = 2 * time.Second - -// ErrIndexLocked is returned when the index is not open within the bounded -// wait — most commonly because another instance holds the single-writer lock. -var ErrIndexLocked = errors.New("search index is not ready; another knowledge-mcp instance may hold the single-writer index lock, retry in a moment") - -// NewIndex opens or creates a bleve index at the given path synchronously. -// indexNames is the set of project names the index will hold; a change from -// the recorded set triggers a rebuild. -func NewIndex(indexPath string, indexNames []string) (*Index, error) { - i := NewLazyIndex(indexPath, indexNames) - i.openStarted = true - if err := i.open(); err != nil { - return nil, err - } - close(i.openDone) - return i, nil -} - -// NewLazyIndex constructs an Index without touching disk. The open is -// deferred until OpenBackground runs it. -func NewLazyIndex(indexPath string, indexNames []string) *Index { - return &Index{ - indexPath: indexPath, - names: indexNames, - ready: make(chan struct{}), - openDone: make(chan struct{}), - } -} - -// OpenBackground starts the index open on a background goroutine exactly -// once. afterOpen runs after a successful open. The open may block for a -// long time if another instance holds the single-writer lock; that must not -// delay the caller (the MCP server) from serving its protocol. -func (i *Index) OpenBackground(afterOpen func()) { - i.openMu.Lock() - if i.openStarted { - i.openMu.Unlock() - return - } - i.openStarted = true - i.openMu.Unlock() - - go func() { - err := i.open() - i.openMu.Lock() - i.openErr = err - i.openMu.Unlock() - close(i.openDone) - if err == nil && afterOpen != nil { - afterOpen() - } - }() -} - -// WaitOpen waits up to d for the open attempt to resolve. It returns nil -// once the index is open, the open error if the attempt failed, or -// ErrIndexLocked if the attempt is still pending after d. -func (i *Index) WaitOpen(d time.Duration) error { - waitCh := func() error { - if i.openErr != nil { - return fmt.Errorf("search index unavailable: %w", i.openErr) - } - return nil - } - - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-i.openDone: - i.openMu.Lock() - defer i.openMu.Unlock() - return waitCh() - case <-timer.C: - select { - case <-i.openDone: - i.openMu.Lock() - defer i.openMu.Unlock() - return waitCh() - default: - return ErrIndexLocked - } - } -} - -// open performs the actual on-disk open or create, exactly as NewIndex -// historically did. -func (i *Index) open() error { - if err := os.MkdirAll(filepath.Dir(i.indexPath), 0755); err != nil { - return fmt.Errorf("create index dir: %w", err) - } - - if metaStale(i.indexPath, i.names) { - if err := DeleteIndex(i.indexPath); err != nil { - return fmt.Errorf("remove stale index: %w", err) - } - } - - mapping := indexMapping() - - var idx bleve.Index - var err error - - if _, statErr := os.Stat(i.indexPath); os.IsNotExist(statErr) { - idx, err = bleve.New(i.indexPath, mapping) - if err != nil { - return fmt.Errorf("create index: %w", err) - } - } else { - idx, err = openOrRecover(i.indexPath, mapping) - if err != nil { - return err - } - } - - meta := indexMeta{Version: metaFileVersion, Names: i.names} - data, marshalErr := json.Marshal(meta) - if marshalErr == nil { - marshalErr = os.WriteFile(metaFilePath(i.indexPath), data, 0644) - } - if marshalErr != nil { - idx.Close() - return fmt.Errorf("write index meta: %w", marshalErr) - } - - i.openMu.Lock() - i.index = idx - i.openMu.Unlock() - - // If index has data, close ready immediately (stale data available) - if i.indexHasData() { - i.CloseReady() - } - - return nil } -func metaFilePath(indexPath string) string { - return indexPath + ".meta.json" -} - -// metaStale reports whether the on-disk index must be rebuilt. -func metaStale(indexPath string, indexNames []string) bool { - data, err := os.ReadFile(metaFilePath(indexPath)) +// NewIndex creates a new in-memory bleve index. +func NewIndex() (*Index, error) { + idx, err := bleve.NewMemOnly(indexMapping()) if err != nil { - return true - } - var meta indexMeta - if err := json.Unmarshal(data, &meta); err != nil { - return true - } - want := slices.Clone(indexNames) - sort.Strings(want) - return meta.Version != metaFileVersion || !slices.Equal(meta.Names, want) -} - -// openOrRecover opens an existing bleve index, recovering from corruption -// (including panics from bbolt on truncated or damaged files) and verifying -// the index is actually usable with a test query. -func openOrRecover(indexPath string, mapping mapping.IndexMapping) (bleve.Index, error) { - var idx bleve.Index - var openErr error - - // bbolt can panic on corrupted files (e.g. truncated root.bolt). - func() { - defer func() { - if r := recover(); r != nil { - openErr = fmt.Errorf("panic opening index: %v", r) - } - }() - idx, openErr = bleve.Open(indexPath) - }() - - if openErr != nil { - log.Printf("warning: index corrupted (%v), rebuilding", openErr) - if rmErr := DeleteIndex(indexPath); rmErr != nil { - return nil, fmt.Errorf("remove corrupted index: %w", rmErr) - } - idx, err := bleve.New(indexPath, mapping) - if err != nil { - return nil, fmt.Errorf("create index after corruption: %w", err) - } - return idx, nil - } - - // Verify the index is actually usable — bolt pages can be - // corrupted in ways that don't surface on Open but hang on use. - testReq := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) - testReq.Size = 1 - if _, searchErr := idx.Search(testReq); searchErr != nil { - log.Printf("warning: index unhealthy (%v), rebuilding", searchErr) - if rmErr := idx.Close(); rmErr != nil { - return nil, fmt.Errorf("close unhealthy index: %w", rmErr) - } - if rmErr := DeleteIndex(indexPath); rmErr != nil { - return nil, fmt.Errorf("remove unhealthy index: %w", rmErr) - } - idx, err := bleve.New(indexPath, mapping) - if err != nil { - return nil, fmt.Errorf("create index after unhealthy: %w", err) - } - return idx, nil - } - - return idx, nil -} - -// indexMapping builds an explicit mapping: text analysis for content fields, -// exact keyword matching for filter fields. -func indexMapping() mapping.IndexMapping { - im := bleve.NewIndexMapping() - doc := bleve.NewDocumentMapping() - for _, f := range []string{"summary", "detail", "rule"} { - fm := bleve.NewTextFieldMapping() - doc.AddFieldMappingsAt(f, fm) - } - for _, f := range []string{"category", "project"} { - fm := bleve.NewTextFieldMapping() - fm.Analyzer = "keyword" - doc.AddFieldMappingsAt(f, fm) + return nil, fmt.Errorf("create in-memory index: %w", err) } - im.DefaultMapping = doc - return im + return &Index{index: idx, ready: make(chan struct{})}, nil } // Add indexes a document with the given (scoped) key. func (i *Index) Add(id string, doc SearchDocument) error { - if err := i.WaitOpen(openWaitTimeout); err != nil { - return err - } return i.index.Index(id, doc) } // Query searches the index, scoped to one project, with full-text // search and optional filters. func (i *Index) Query(project, q, category string, limit int) ([]SearchResult, error) { - if err := i.WaitOpen(openWaitTimeout); err != nil { - return nil, err - } - - // Block until indexing completes if no stale data available + // Block until indexing completes. <-i.ready if limit <= 0 { @@ -386,15 +137,12 @@ func (i *Index) Query(project, q, category string, limit int) ([]SearchResult, e return results, nil } -// Close closes the bleve index, if it was ever opened. +// Close closes the bleve index. func (i *Index) Close() error { - i.openMu.Lock() - idx := i.index - i.openMu.Unlock() - if idx == nil { + if i == nil || i.index == nil { return nil } - return idx.Close() + return i.index.Close() } // CloseReady closes the ready channel exactly once, preventing double-close panics. @@ -403,18 +151,7 @@ func (i *Index) CloseReady() { i.readyOnce.Do(func() { close(i.ready) }) } -// indexHasData reports whether the index contains any documents. -func (i *Index) indexHasData() bool { - req := bleve.NewSearchRequest(bleve.NewMatchAllQuery()) - req.Size = 0 - result, err := i.index.Search(req) - if err != nil { - return false - } - return result.Total > 0 -} - -// WaitReady blocks until the index has data available for queries. +// WaitReady blocks until the index is ready for queries. func (i *Index) WaitReady() { <-i.ready } @@ -424,8 +161,8 @@ func (i *Index) DocCount() (uint64, error) { return i.index.DocCount() } -// IndexAll indexes all projects known to the resolver in the background. -// It closes the ready channel when complete. +// IndexAll indexes all projects known to the resolver. It closes the ready +// channel when complete. func (i *Index) IndexAll(res *projects.Resolver) { defer i.CloseReady() @@ -491,9 +228,22 @@ func (i *Index) indexOrgKnowledge(knowledgeDir, orgName string) { } } -// DeleteIndex removes a bleve index directory. -func DeleteIndex(indexPath string) error { - return os.RemoveAll(indexPath) +// indexMapping builds an explicit mapping: text analysis for content fields, +// exact keyword matching for filter fields. +func indexMapping() mapping.IndexMapping { + im := bleve.NewIndexMapping() + doc := bleve.NewDocumentMapping() + for _, f := range []string{"summary", "detail", "rule"} { + fm := bleve.NewTextFieldMapping() + doc.AddFieldMappingsAt(f, fm) + } + for _, f := range []string{"category", "project"} { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "keyword" + doc.AddFieldMappingsAt(f, fm) + } + im.DefaultMapping = doc + return im } // escapeQueryString escapes lucene/bleve query-string syntax characters so diff --git a/search/index_test.go b/search/index_test.go index 1c6c490..e8c40d9 100644 --- a/search/index_test.go +++ b/search/index_test.go @@ -1,7 +1,6 @@ package search import ( - "bytes" "os" "path/filepath" "testing" @@ -10,9 +9,9 @@ import ( "github.com/renderorange/knowledge-mcp/projects" ) -func newIndex(t *testing.T, path string, names []string) *Index { +func newIndex(t *testing.T) *Index { t.Helper() - idx, err := NewIndex(path, names) + idx, err := NewIndex() if err != nil { t.Fatalf("NewIndex() error: %v", err) } @@ -21,16 +20,15 @@ func newIndex(t *testing.T, path string, names []string) *Index { // newIndexReady creates an index and closes the ready channel immediately, // for unit tests that add documents directly and query without IndexAll. -func newIndexReady(t *testing.T, path string, names []string) *Index { +func newIndexReady(t *testing.T) *Index { t.Helper() - idx := newIndex(t, path, names) + idx := newIndex(t) idx.CloseReady() return idx } func TestIndexCreateAndQuery(t *testing.T) { - dir := t.TempDir() - idx := newIndexReady(t, filepath.Join(dir, "test.bleve"), []string{"test"}) + idx := newIndexReady(t) defer idx.Close() docs := []struct { @@ -118,9 +116,7 @@ func TestIndexCreateAndQuery(t *testing.T) { } func TestProjectScopingNoCrossProjectLeakage(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - idx := newIndexReady(t, indexPath, []string{"alpha", "beta"}) + idx := newIndexReady(t) defer idx.Close() // Same bare entry ID in two projects — must not overwrite each other. @@ -163,64 +159,8 @@ func TestProjectScopingNoCrossProjectLeakage(t *testing.T) { } } -func TestIndexReopen(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - idx := newIndex(t, indexPath, []string{"test"}) - if err := idx.Add("test/conv-001", SearchDocument{Summary: "persist", Project: "test"}); err != nil { - t.Fatalf("Add() error: %v", err) - } - idx.Close() - - idx2 := newIndex(t, indexPath, []string{"test"}) - defer idx2.Close() - - results, err := idx2.Query("test", "persist", "", 10) - if err != nil { - t.Fatalf("Query() error: %v", err) - } - if len(results) != 1 { - t.Errorf("len(results) = %d, want 1", len(results)) - } -} - -func TestIndexRebuildOnNameSetChange(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - idx := newIndex(t, indexPath, []string{"alpha"}) - if err := idx.Add("alpha/conv-001", SearchDocument{Summary: "alpha doc", Project: "alpha"}); err != nil { - t.Fatalf("Add() error: %v", err) - } - idx.Close() - - // Same name set: no rebuild, doc persists. - idx2 := newIndex(t, indexPath, []string{"alpha"}) - results, err := idx2.Query("alpha", "", "", 10) - if err != nil { - t.Fatalf("Query() error: %v", err) - } - if len(results) != 1 { - t.Fatalf("len(results) = %d, want 1 (index was needlessly rebuilt)", len(results)) - } - idx2.Close() - - // Different name set: rebuild, old docs gone. - idx3 := newIndexReady(t, indexPath, []string{"alpha", "beta"}) - defer idx3.Close() - results, err = idx3.Query("alpha", "", "", 10) - if err != nil { - t.Fatalf("Query() error: %v", err) - } - if len(results) != 0 { - t.Fatalf("len(results) = %d, want 0 (index should have been rebuilt)", len(results)) - } -} - func TestIndexEmptyQuery(t *testing.T) { - dir := t.TempDir() - idx := newIndexReady(t, filepath.Join(dir, "test.bleve"), []string{"test"}) + idx := newIndexReady(t) defer idx.Close() if err := idx.Add("test/test-001", SearchDocument{ @@ -239,8 +179,7 @@ func TestIndexEmptyQuery(t *testing.T) { } func TestQueryEscapesSpecialCharacters(t *testing.T) { - dir := t.TempDir() - idx := newIndexReady(t, filepath.Join(dir, "test.bleve"), []string{"test"}) + idx := newIndexReady(t) defer idx.Close() if err := idx.Add("test/conv-001", SearchDocument{ @@ -276,91 +215,39 @@ func TestQueryEscapesSpecialCharacters(t *testing.T) { } } -func TestIndexRecoverFromCorruption(t *testing.T) { +// TestMemoryIndexCreatesNoFiles pins the memory-only contract: creating, +// adding to, and closing an index must never touch the filesystem. +func TestMemoryIndexCreatesNoFiles(t *testing.T) { + stateHome := t.TempDir() + t.Setenv("XDG_STATE_HOME", stateHome) dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - // Create index and add a document - idx := newIndex(t, indexPath, []string{"test"}) - if err := idx.Add("test/conv-001", SearchDocument{ - Summary: "original doc", Project: "test", - }); err != nil { + idx := newIndexReady(t) + if err := idx.Add("test/conv-001", SearchDocument{Summary: "doc", Project: "test"}); err != nil { t.Fatalf("Add() error: %v", err) } - idx.Close() - - // Corrupt the index by deleting a segment file - storeDir := filepath.Join(indexPath, "store") - entries, err := os.ReadDir(storeDir) - if err != nil { - t.Fatalf("ReadDir(%s) error: %v", storeDir, err) - } - if len(entries) == 0 { - t.Fatal("no files in store directory") - } - // Delete the first segment file to corrupt the index - for _, e := range entries { - if e.Name() != "root.bolt" { - os.Remove(filepath.Join(storeDir, e.Name())) - break - } + if err := idx.Close(); err != nil { + t.Fatalf("Close() error: %v", err) } - // Opening should recover from corruption, not fail - idx2 := newIndexReady(t, indexPath, []string{"test"}) - defer idx2.Close() - - // Old data is gone (rebuilt), but server should work - results, err := idx2.Query("test", "", "", 10) - if err != nil { - t.Fatalf("Query() after recovery error: %v", err) - } - if len(results) != 0 { - t.Errorf("len(results) = %d, want 0 (old data should be gone after rebuild)", len(results)) - } - - // Can add new data after recovery - if err := idx2.Add("test/conv-002", SearchDocument{ - Summary: "new doc after recovery", Project: "test", - }); err != nil { - t.Fatalf("Add() after recovery error: %v", err) - } - results, err = idx2.Query("test", "recovery", "", 10) + entries, err := os.ReadDir(dir) if err != nil { - t.Fatalf("Query() after recovery error: %v", err) - } - if len(results) != 1 { - t.Errorf("len(results) = %d, want 1", len(results)) - } -} - -func TestIndexHasData(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - idx := newIndex(t, indexPath, []string{"test"}) - defer idx.Close() - if idx.indexHasData() { - t.Error("empty index should not have data") + t.Fatalf("ReadDir() error: %v", err) } - - if err := idx.Add("test/conv-001", SearchDocument{Summary: "test", Project: "test"}); err != nil { - t.Fatalf("Add() error: %v", err) + if len(entries) != 0 { + t.Errorf("index created %d artifacts in the working temp dir, want 0", len(entries)) } - - if !idx.indexHasData() { - t.Error("index with data should have data") + if _, err := os.Stat(filepath.Join(stateHome, "knowledge-mcp")); !os.IsNotExist(err) { + t.Errorf("index created artifacts under XDG_STATE_HOME: %v", err) } } func TestWaitReadyBlocksWhenEmpty(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - idx := newIndex(t, indexPath, []string{"test"}) + idx := newIndex(t) defer idx.Close() - // ready channel should be open (blocking) for empty index + // ready channel should be open (blocking) for an index that has not + // completed IndexAll or CloseReady. select { case <-idx.ready: t.Error("ready channel should not be closed for empty index") @@ -369,33 +256,10 @@ func TestWaitReadyBlocksWhenEmpty(t *testing.T) { } } -func TestWaitReadyReturnsImmediatelyWithData(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - idx := newIndex(t, indexPath, []string{"test"}) - if err := idx.Add("test/conv-001", SearchDocument{Summary: "test", Project: "test"}); err != nil { - t.Fatalf("Add() error: %v", err) - } - idx.Close() - - // Reopen — index has data, ready should be closed immediately - idx2 := newIndex(t, indexPath, []string{"test"}) - defer idx2.Close() - - select { - case <-idx2.ready: - // expected: channel is closed (stale data available) - default: - t.Error("ready channel should be closed for index with stale data") - } -} - func TestIndexAllBackground(t *testing.T) { dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - idx := newIndex(t, indexPath, []string{"test-project"}) + idx := newIndex(t) defer idx.Close() // Create a mock knowledge file @@ -442,32 +306,14 @@ func TestIndexAllBackground(t *testing.T) { } } -func TestIndexAllWithExistingDataNoDoubleClosePanic(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - // Create index and add a document, then close. - idx := newIndex(t, indexPath, []string{"test-project"}) - if err := idx.Add("test-project/conv-001", SearchDocument{ - Summary: "persisted doc", Project: "test-project", - }); err != nil { - t.Fatalf("Add() error: %v", err) - } - idx.Close() - - // Reopen — NewIndex will close ready because index has data. - idx2 := newIndex(t, indexPath, []string{"test-project"}) - defer idx2.Close() - - // Verify ready is already closed. - select { - case <-idx2.ready: - // expected - default: - t.Fatal("ready channel should be closed for index with stale data") - } +// TestIndexAllAfterCloseReadyNoPanic covers the ready-channel double-close +// path: IndexAll must complete without panicking when ready was already +// closed by CloseReady. +func TestIndexAllAfterCloseReadyNoPanic(t *testing.T) { + idx := newIndexReady(t) + defer idx.Close() - // Create a resolver so IndexAll can run. + dir := t.TempDir() agentsDir := filepath.Join(dir, "test-project", ".agents") if err := os.MkdirAll(agentsDir, 0755); err != nil { t.Fatalf("MkdirAll() error: %v", err) @@ -490,7 +336,7 @@ func TestIndexAllWithExistingDataNoDoubleClosePanic(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - idx2.IndexAll(resolver) + idx.IndexAll(resolver) }() select { @@ -500,7 +346,7 @@ func TestIndexAllWithExistingDataNoDoubleClosePanic(t *testing.T) { t.Fatal("IndexAll did not complete within timeout") } - results, err := idx2.Query("test-project", "convention", "", 10) + results, err := idx.Query("test-project", "convention", "", 10) if err != nil { t.Fatalf("Query() error: %v", err) } @@ -511,10 +357,9 @@ func TestIndexAllWithExistingDataNoDoubleClosePanic(t *testing.T) { func TestBackgroundIndexingIntegration(t *testing.T) { dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") // Create index with no data (ready channel should be open) - idx := newIndex(t, indexPath, []string{"test"}) + idx := newIndex(t) // Verify ready is blocking select { @@ -582,53 +427,3 @@ func TestBackgroundIndexingIntegration(t *testing.T) { idx.Close() } - -func TestIndexRecoverFromSilentCorruption(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "test.bleve") - - // Create index and add a document - idx := newIndex(t, indexPath, []string{"test"}) - if err := idx.Add("test/conv-001", SearchDocument{ - Summary: "original doc", Project: "test", - }); err != nil { - t.Fatalf("Add() error: %v", err) - } - idx.Close() - - // Overwrite root.bolt with garbage — simulates a partially written - // or corrupted file where bleve.Open() may succeed but the index - // is internally broken. We overwrite rather than truncate because - // truncating an mmap'd file causes SIGBUS which is not recoverable. - boltPath := filepath.Join(indexPath, "store", "root.bolt") - if err := os.WriteFile(boltPath, bytes.Repeat([]byte{0xff}, 4096), 0600); err != nil { - t.Fatalf("WriteFile() error: %v", err) - } - - // Opening should detect the unhealthy index and rebuild - idx2 := newIndexReady(t, indexPath, []string{"test"}) - defer idx2.Close() - - // Old data is gone (rebuilt), but server should work - results, err := idx2.Query("test", "", "", 10) - if err != nil { - t.Fatalf("Query() after recovery error: %v", err) - } - if len(results) != 0 { - t.Errorf("len(results) = %d, want 0 (old data should be gone after rebuild)", len(results)) - } - - // Can add new data after recovery - if err := idx2.Add("test/conv-003", SearchDocument{ - Summary: "new doc after silent recovery", Project: "test", - }); err != nil { - t.Fatalf("Add() after recovery error: %v", err) - } - results, err = idx2.Query("test", "silent", "", 10) - if err != nil { - t.Fatalf("Query() after recovery error: %v", err) - } - if len(results) != 1 { - t.Errorf("len(results) = %d, want 1", len(results)) - } -} diff --git a/search/loadwarn_test.go b/search/loadwarn_test.go index dae23a6..a91ef76 100644 --- a/search/loadwarn_test.go +++ b/search/loadwarn_test.go @@ -53,7 +53,7 @@ func runIndexAllCapture(t *testing.T, dir string, files map[string]string) (stri t.Fatalf("BuildWithStore() error: %v", err) } - idx := newIndex(t, filepath.Join(dir, "test.bleve"), []string{"test-project"}) + idx := newIndex(t) var logBuf syncBuffer log.SetOutput(&logBuf) diff --git a/search/location.go b/search/location.go deleted file mode 100644 index f295c79..0000000 --- a/search/location.go +++ /dev/null @@ -1,39 +0,0 @@ -package search - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" - "os" - "path/filepath" - "strings" -) - -// Location picks the bleve index path: explicit override, a --store -// default (/.index), legacy per-config locations for single-entry -// configs, or a hashed XDG state dir for multi-entry configs. -func Location(override, store string, roots, projs, canonicalEntries []string) (string, error) { - if override != "" { - return filepath.Abs(override) - } - if store != "" { - return filepath.Join(store, ".index"), nil - } - if len(roots)+len(projs) == 1 { - single := roots - if len(single) == 0 { - single = projs - } - return filepath.Join(single[0], ".agents", ".index"), nil - } - base := os.Getenv("XDG_STATE_HOME") - if base == "" { - home, err := os.UserHomeDir() - if err != nil { - return "", fmt.Errorf("resolve state dir: %w", err) - } - base = filepath.Join(home, ".local", "state") - } - sum := sha256.Sum256([]byte(strings.Join(canonicalEntries, "\x00"))) - return filepath.Join(base, "knowledge-mcp", hex.EncodeToString(sum[:8])+".index"), nil -} diff --git a/search/location_test.go b/search/location_test.go deleted file mode 100644 index dbb0c2e..0000000 --- a/search/location_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package search - -import ( - "os" - "path/filepath" - "strings" - "testing" -) - -func TestLocationOverride(t *testing.T) { - p, err := Location("/custom/index", "", []string{}, []string{}, nil) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if p != "/custom/index" { - t.Errorf("location = %q, want %q", p, "/custom/index") - } -} - -func TestLocationLegacySingleProject(t *testing.T) { - p, err := Location("", "", []string{}, []string{"/proj"}, nil) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if want := filepath.Join("/proj", ".agents", ".index"); p != want { - t.Errorf("location = %q, want %q", p, want) - } -} - -func TestLocationLegacySingleRoot(t *testing.T) { - p, err := Location("", "", []string{"/root"}, []string{}, nil) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if want := filepath.Join("/root", ".agents", ".index"); p != want { - t.Errorf("location = %q, want %q", p, want) - } -} - -func TestLocationMultiEntryStateDir(t *testing.T) { - t.Setenv("XDG_STATE_HOME", t.TempDir()) - - entries := []string{"/rootB", "/rootA", "/projC"} - p, err := Location("", "", []string{"/rootB", "/rootA"}, []string{"/projC"}, entries) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if !strings.Contains(p, "knowledge-mcp") { - t.Errorf("location = %q, want under knowledge-mcp state dir", p) - } - - // Same entries -> same location (deterministic). - p2, err := Location("", "", []string{"/rootA", "/rootB"}, []string{"/projC"}, entries) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if p != p2 { - t.Errorf("location not deterministic: %q vs %q", p, p2) - } - - // Different entries -> different location. - entries2 := []string{"/rootB", "/rootA", "/projD"} - p3, err := Location("", "", []string{"/rootB", "/rootA"}, []string{"/projD"}, entries2) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if p == p3 { - t.Errorf("different entries should hash to different locations: %q", p) - } -} - -func TestLocationXDGDefault(t *testing.T) { - t.Setenv("XDG_STATE_HOME", "") - home, err := os.UserHomeDir() - if err != nil { - t.Skip("no home dir") - } - p, err := Location("", "", []string{"/a", "/b"}, []string{}, []string{"/a", "/b"}) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if want := filepath.Join(home, ".local", "state", "knowledge-mcp"); !strings.HasPrefix(p, want) { - t.Errorf("location = %q, want under %q", p, want) - } -} - -func TestLocationStoreDefault(t *testing.T) { - p, err := Location("", "/store", []string{"/root"}, []string{}, nil) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if want := filepath.Join("/store", ".index"); p != want { - t.Errorf("location = %q, want %q", p, want) - } -} - -func TestLocationStoreMultiEntry(t *testing.T) { - p, err := Location("", "/store", []string{"/a", "/b"}, []string{"/c"}, []string{"/a", "/b", "/c"}) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if want := filepath.Join("/store", ".index"); p != want { - t.Errorf("location = %q, want %q", p, want) - } -} - -func TestLocationOverrideBeatsStore(t *testing.T) { - p, err := Location("/custom/index", "/store", []string{}, []string{}, nil) - if err != nil { - t.Fatalf("Location() error: %v", err) - } - if p != "/custom/index" { - t.Errorf("location = %q, want %q", p, "/custom/index") - } -} diff --git a/search/race_test.go b/search/race_test.go deleted file mode 100644 index 2c2f1d5..0000000 --- a/search/race_test.go +++ /dev/null @@ -1,125 +0,0 @@ -package search - -import ( - "path/filepath" - "testing" - "time" -) - -// TestOpenBackgroundDoesNotBlockWhileLocked covers the startup race: a second -// server instance must not wedge the MCP handshake when the first instance -// holds the single-writer index lock (bbolt flocks block indefinitely). -func TestOpenBackgroundDoesNotBlockWhileLocked(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "shared.index") - - // Instance A creates the index and holds the bbolt lock. - a := newIndexReady(t, indexPath, []string{"test"}) - if err := a.Add("test/conv-001", SearchDocument{Summary: "persisted doc", Project: "test"}); err != nil { - t.Fatalf("Add() to instance A: %v", err) - } - - // Instance B attempts an async open; it must not block this goroutine. - b := NewLazyIndex(indexPath, []string{"test"}) - b.OpenBackground(nil) - defer b.Close() - - if err := b.WaitOpen(250 * time.Millisecond); err == nil { - t.Fatal("WaitOpen() resolved while the index lock was still held, want unresolved") - } - - // Once A releases the lock, B's pending open must complete on its own. - if err := a.Close(); err != nil { - t.Fatalf("closing instance A: %v", err) - } - if err := b.WaitOpen(10 * time.Second); err != nil { - t.Fatalf("WaitOpen() after lock release: %v", err) - } - - results, err := b.Query("test", "persisted", "", 10) - if err != nil { - t.Fatalf("Query() after recovery: %v", err) - } - if len(results) != 1 { - t.Fatalf("Query() returned %d results after recovery, want 1", len(results)) - } -} - -// TestQueryDoesNotHangWhileLocked covers tool calls: a query against a -// locked index must return an error promptly instead of blocking forever. -func TestQueryDoesNotHangWhileLocked(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "shared.index") - - a := newIndexReady(t, indexPath, []string{"test"}) - if err := a.Add("test/conv-001", SearchDocument{Summary: "locked doc", Project: "test"}); err != nil { - t.Fatalf("Add() to instance A: %v", err) - } - - b := NewLazyIndex(indexPath, []string{"test"}) - b.OpenBackground(nil) - defer b.Close() - - done := make(chan error, 1) - go func() { - _, err := b.Query("test", "locked", "", 10) - done <- err - }() - - select { - case err := <-done: - if err == nil { - t.Fatal("Query() succeeded against a locked index, want error") - } - case <-time.After(5 * time.Second): - t.Fatal("Query() hung while the index was locked, want prompt error") - } - - // After the lock is released the same query must succeed, proving the - // instance recovers without a restart. - if err := a.Close(); err != nil { - t.Fatalf("closing instance A: %v", err) - } - if err := b.WaitOpen(10 * time.Second); err != nil { - t.Fatalf("WaitOpen() after lock release: %v", err) - } - results, err := b.Query("test", "locked", "", 10) - if err != nil { - t.Fatalf("Query() after recovery: %v", err) - } - if len(results) != 1 { - t.Fatalf("Query() returned %d results after recovery, want 1", len(results)) - } -} - -// TestAddDoesNotPanicWhileLocked covers writes: Add must surface the open -// state as an error, never dereference a nil index. -func TestAddDoesNotPanicWhileLocked(t *testing.T) { - dir := t.TempDir() - indexPath := filepath.Join(dir, "shared.index") - - a := newIndexReady(t, indexPath, []string{"test"}) - - b := NewLazyIndex(indexPath, []string{"test"}) - b.OpenBackground(nil) - defer b.Close() - - errCh := make(chan error, 1) - go func() { - errCh <- b.Add("test/conv-002", SearchDocument{Summary: "new doc", Project: "test"}) - }() - - select { - case err := <-errCh: - if err == nil { - t.Fatal("Add() succeeded against a locked index, want error") - } - case <-time.After(5 * time.Second): - t.Fatal("Add() hung while the index was locked, want prompt error") - } - - if err := a.Close(); err != nil { - t.Fatalf("closing instance A: %v", err) - } - _ = b.WaitOpen(10 * time.Second) -} From 0dea3ab50966eb73dfa9c4b10508cebd605d6b3c Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 10:49:54 -0400 Subject: [PATCH 2/6] GH-34: adopt in-memory index in tool tests --- tools/integration_test.go | 10 ++++----- tools/tools_test.go | 44 +++++++++++++++------------------------ 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/tools/integration_test.go b/tools/integration_test.go index 76d84d9..bfb0108 100644 --- a/tools/integration_test.go +++ b/tools/integration_test.go @@ -24,8 +24,7 @@ func TestEndToEnd(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(dir, ".index") - idx := newIndexReady(t, indexPath, []string{projectName}) + idx := newIndexReady(t) defer idx.Close() // Step 1: Init @@ -230,8 +229,7 @@ func TestOrgWideMode(t *testing.T) { initReqB.Params.Arguments = map[string]interface{}{"project_path": projB} InitHandler(resolver)(context.Background(), initReqB) - indexPath := filepath.Join(orgDir, ".index") - idx, err := search.NewIndex(indexPath, []string{"projectA", "projectB"}) + idx, err := search.NewIndex() if err != nil { t.Fatalf("NewIndex() error: %v", err) } @@ -298,7 +296,7 @@ func TestFullWorkflow(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - idx := newIndexReady(t, filepath.Join(orgRoot, ".agents", ".index"), []string{"app", "lib"}) + idx := newIndexReady(t) defer idx.Close() // 1. List projects @@ -382,7 +380,7 @@ func TestStoreModeEndToEnd(t *testing.T) { t.Errorf("expected in-tree ignore warning, got %v", warnings) } - idx := newIndexReady(t, filepath.Join(store, ".index"), []string{"proj"}) + idx := newIndexReady(t) defer idx.Close() // Init writes into the store. diff --git a/tools/tools_test.go b/tools/tools_test.go index 4cfc030..fa8c3fd 100644 --- a/tools/tools_test.go +++ b/tools/tools_test.go @@ -15,11 +15,12 @@ import ( "github.com/renderorange/knowledge-mcp/search" ) -// newIndexReady creates an index and closes the ready channel immediately, -// for unit tests that add documents directly and query without IndexAll. -func newIndexReady(t *testing.T, path string, names []string) *search.Index { +// newIndexReady creates an in-memory index and closes the ready channel +// immediately, for unit tests that add documents directly and query without +// IndexAll. +func newIndexReady(t *testing.T) *search.Index { t.Helper() - idx, err := search.NewIndex(path, names) + idx, err := search.NewIndex() if err != nil { t.Fatalf("search.NewIndex() error: %v", err) } @@ -86,8 +87,7 @@ func TestWriteHandler(t *testing.T) { kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: []knowledge.Entry{}} knowledge.Save(knowledge.CategoryFilePath(filepath.Join(dir, ".agents"), "conventions"), kf) - indexPath := filepath.Join(dir, ".index") - idx, _ := search.NewIndex(indexPath, []string{"test"}) + idx, _ := search.NewIndex() defer idx.Close() resolver, _, err := projects.Build([]string{root}, nil, "") @@ -221,8 +221,7 @@ func TestQueryHandler(t *testing.T) { kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: []knowledge.Entry{}} knowledge.Save(knowledge.CategoryFilePath(agentsDir, "conventions"), kf) - indexPath := filepath.Join(dir, ".index") - idx := newIndexReady(t, indexPath, []string{"test"}) + idx := newIndexReady(t) defer idx.Close() resolver, _, err := projects.Build([]string{root}, nil, "") @@ -301,8 +300,7 @@ func TestListHandler(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{"test"}) + idx := newIndexReady(t) defer idx.Close() handler := ListHandler(resolver, idx) @@ -376,8 +374,7 @@ func TestUpdateHandler(t *testing.T) { kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: []knowledge.Entry{entry}} knowledge.Save(knowledge.CategoryFilePath(agentsDir, "conventions"), kf) - indexPath := filepath.Join(dir, ".index") - idx, _ := search.NewIndex(indexPath, []string{"test"}) + idx, _ := search.NewIndex() defer idx.Close() resolver, _, err := projects.Build([]string{root}, nil, "") @@ -710,7 +707,7 @@ func TestQueryProjectIsolation(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - idx := newIndexReady(t, filepath.Join(orgDir, ".index"), []string{"projectA", "projectB"}) + idx := newIndexReady(t) defer idx.Close() write := WriteHandler(resolver, idx) @@ -797,8 +794,7 @@ func TestQueryWithGlobalMerge(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{"myproj", "_global"}) + idx := newIndexReady(t) defer idx.Close() // Write to project @@ -876,8 +872,7 @@ func TestQueryWithoutGlobal(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{"myproj"}) + idx := newIndexReady(t) defer idx.Close() write := WriteHandler(resolver, idx) @@ -913,8 +908,7 @@ func TestWriteWithRule(t *testing.T) { kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: []knowledge.Entry{}} knowledge.Save(knowledge.CategoryFilePath(agentsDir, "conventions"), kf) - indexPath := filepath.Join(dir, ".index") - idx := newIndexReady(t, indexPath, []string{"test"}) + idx := newIndexReady(t) defer idx.Close() resolver, _, err := projects.Build([]string{root}, nil, "") @@ -1013,8 +1007,7 @@ func TestUpdateWithRule(t *testing.T) { kf := &knowledge.KnowledgeFile{Project: "test", Version: 1, Entries: []knowledge.Entry{entry}} knowledge.Save(knowledge.CategoryFilePath(agentsDir, "conventions"), kf) - indexPath := filepath.Join(dir, ".index") - idx, _ := search.NewIndex(indexPath, []string{"test"}) + idx, _ := search.NewIndex() defer idx.Close() resolver, _, err := projects.Build([]string{root}, nil, "") @@ -1062,8 +1055,7 @@ func TestListGlobalProvenance(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{"myproj", "_global"}) + idx := newIndexReady(t) defer idx.Close() handler := ListHandler(resolver, idx) @@ -1096,8 +1088,7 @@ func TestListOrgGroupedByFile(t *testing.T) { } rootName := filepath.Base(root) - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{rootName}) + idx := newIndexReady(t) defer idx.Close() for _, sec := range []struct{ file, heading, detail string }{ @@ -1149,8 +1140,7 @@ func TestListConstraintsSeparation(t *testing.T) { t.Fatalf("projects.Build() error: %v", err) } - indexPath := filepath.Join(root, ".index") - idx := newIndexReady(t, indexPath, []string{"test"}) + idx := newIndexReady(t) defer idx.Close() handler := ListHandler(resolver, idx) From 0a2fcb2c1431178f0532d597c3eaf491e911c858 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 11:03:25 -0400 Subject: [PATCH 3/6] GH-34: serve queries from an in-memory index --- main.go | 43 +++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/main.go b/main.go index 89f28af..e0b339f 100644 --- a/main.go +++ b/main.go @@ -65,9 +65,9 @@ func main() { var projs pathList showVersion := flag.Bool("version", false, "Print version and exit") globalPath := flag.String("global", "", "Path to a global knowledge store shared across all projects") - indexOverride := flag.String("index", "", "Override the search index location") + indexOverride := flag.String("index", "", "Deprecated: the search index is in-memory; accepted and ignored") storeDir := flag.String("store", "", "Central directory for all knowledge stores; in-tree .agents/ is ignored when set") - noIndexOnStartup := flag.Bool("no-index-on-startup", false, "Skip indexing on startup; index may be stale or empty") + noIndexOnStartup := flag.Bool("no-index-on-startup", false, "Deprecated: the index is always built at startup; accepted and ignored") debugMode := flag.Bool("debug", false, "Log diagnostics to stderr (and --log-file if set); lands in the client's log") logFile := flag.String("log-file", "", "Tee debug lines to this file (implies --debug)") flag.Var(&roots, "root", "Org root whose immediate children are projects (repeatable)") @@ -111,32 +111,27 @@ func main() { dbg.Debugf("startup", "resolve.done", "entries", len(resolver.Entries()), "warnings", len(warnings), "dur", time.Since(tResolve).Round(time.Millisecond)) tIndex := time.Now() - indexBasePath, err := search.Location(*indexOverride, *storeDir, []string(roots), []string(projs), resolver.Entries()) - if err != nil { - log.Fatalf("determine index location: %v", err) + if *indexOverride != "" { + log.Printf("warning: --index is deprecated and ignored; the search index is per-process in-memory") + } + if *noIndexOnStartup { + log.Printf("warning: --no-index-on-startup is deprecated and ignored; the index is always built at startup") } - idx := search.NewLazyIndex(indexBasePath, resolver.KnownNames()) + idx, err := search.NewIndex() + if err != nil { + log.Fatalf("create search index: %v", err) + } defer idx.Close() - // Open the index in the background: a second instance whose index is - // locked by another knowledge-mcp (single-writer) must not delay the - // MCP protocol. Tool calls degrade to a clear error until the lock - // frees, after which the pending open completes on its own. - var afterOpen func() - if !*noIndexOnStartup { - afterOpen = func() { - dbg.Debugf("startup", "index.open", "path", indexBasePath, "dur", time.Since(tIndex).Round(time.Millisecond)) - dbg.Debugf("startup", "index.kick", "names", len(resolver.KnownNames())) - idx.IndexAll(resolver) - } - } else { - dbg.Debugf("startup", "index.kick", "skipped", "no-index-on-startup") - afterOpen = func() { - dbg.Debugf("startup", "index.open", "path", indexBasePath, "dur", time.Since(tIndex).Round(time.Millisecond)) - } - } - idx.OpenBackground(afterOpen) + // Kick background indexing once; queries block on the ready channel + // until IndexAll completes (same cold-start semantics as the former + // on-disk index). With no disk index there is no cross-instance lock + // to contend on. + go func() { + dbg.Debugf("startup", "index.kick", "names", len(resolver.KnownNames()), "dur", time.Since(tIndex).Round(time.Millisecond)) + idx.IndexAll(resolver) + }() // Create MCP server s := server.NewMCPServer( From 1f6bd5486e2add5fafcd4bbbc911dd7fb69ddb12 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 11:13:14 -0400 Subject: [PATCH 4/6] GH-34: drop disk-index diagnostics from doctor --- internal/diag/doctor.go | 150 +++------------------ internal/diag/doctor_test.go | 248 +++++------------------------------ 2 files changed, 50 insertions(+), 348 deletions(-) diff --git a/internal/diag/doctor.go b/internal/diag/doctor.go index a3af38b..e1d9bd0 100644 --- a/internal/diag/doctor.go +++ b/internal/diag/doctor.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "path/filepath" "sort" "strings" "time" @@ -17,7 +16,7 @@ import ( ) // ErrQueryBlocked is returned by watchdog when fn exceeds the deadline — -// the cold-index query-block detector. +// the slow-kick query-block detector. var ErrQueryBlocked = errors.New("query blocked: index never became ready") // doctorPaths collects repeatable path flags (mirror of main's pathList). @@ -46,9 +45,9 @@ func RunDoctor(args []string, version string) int { fs := flag.NewFlagSet("debug", flag.ContinueOnError) var roots, projs doctorPaths globalPath := fs.String("global", "", "Path to a global knowledge store shared across all projects") - indexOverride := fs.String("index", "", "Override the search index location") + indexOverride := fs.String("index", "", "Deprecated: the search index is in-memory; accepted and ignored") storeDir := fs.String("store", "", "Central directory for all knowledge stores") - noIndexOnStartup := fs.Bool("no-index-on-startup", false, "Skip indexing before the self-query") + noIndexOnStartup := fs.Bool("no-index-on-startup", false, "Deprecated: the index is always built; accepted and ignored") timeout := fs.Duration("timeout", 5*time.Second, "Self-query watchdog deadline") logFile := fs.String("log-file", "", "Tee the report to this file") fs.Var(&roots, "root", "Org root whose immediate children are projects (repeatable)") @@ -100,61 +99,26 @@ func RunDoctor(args []string, version string) int { } fmt.Fprintf(out, "entries=%d warnings=%d\n", len(resolver.Entries()), len(warnings)) - tIndex := time.Now() - indexPath, err := search.Location(*indexOverride, *storeDir, []string(roots), []string(projs), resolver.Entries()) - if err != nil { - add("fail", "index.location", err.Error()) - printFindings(out, findings) - return 1 - } - indexSource := "hashed state dir" if *indexOverride != "" { - indexSource = "override" - } else if *storeDir != "" { - indexSource = "store" - } else if len(roots)+len(projs) == 1 { - indexSource = "single config" + fmt.Fprintln(out, "warning: --index is deprecated and ignored; the search index is per-process in-memory") + } + if *noIndexOnStartup { + fmt.Fprintln(out, "warning: --no-index-on-startup is deprecated and ignored; the index is always built") } - idx := search.NewLazyIndex(indexPath, resolver.KnownNames()) - - var indexDone chan struct{} - defer func() { - if indexDone == nil { - idx.Close() - return - } - select { - case <-indexDone: - idx.Close() - default: - // IndexAll may still be in flight; leaking is safe, process exits. - } - }() - - idx.OpenBackground(nil) - err = idx.WaitOpen(*timeout) - indexDur := time.Since(tIndex).Round(time.Millisecond) + tIndex := time.Now() + idx, err := search.NewIndex() if err != nil { - if errors.Is(err, search.ErrIndexLocked) { - add("fail", "index.locked", "another knowledge-mcp instance may hold the single-writer index lock") - } else { - add("fail", "index.open", err.Error()) - } + add("fail", "index.create", err.Error()) printFindings(out, findings) return 1 } + defer idx.Close() fmt.Fprintln(out, "\n== Startup trail ==") fmt.Fprintf(out, "resolve.done dur=%s\n", resolveDur) - fmt.Fprintf(out, "index.open path=%s source=%s dur=%s\n", indexPath, indexSource, indexDur) - if *noIndexOnStartup { - fmt.Fprintln(out, "index.kick skipped=no-index-on-startup") - } else { - fmt.Fprintln(out, "index.kick done") - indexDone = make(chan struct{}) - go func() { idx.IndexAll(resolver); close(indexDone) }() - } + fmt.Fprintf(out, "index.kick done (in-memory, dur=%s)\n", time.Since(tIndex).Round(time.Millisecond)) + go idx.IndexAll(resolver) tQuery := time.Now() qerr := watchdog(*timeout, func() error { @@ -164,13 +128,8 @@ func RunDoctor(args []string, version string) int { queryDur := time.Since(tQuery).Round(time.Millisecond) switch { case errors.Is(qerr, ErrQueryBlocked): - if *noIndexOnStartup { - add("warn", "query.blocked", "reason=no-index-on-startup") - fmt.Fprintf(out, "self-query warn query.blocked reason=no-index-on-startup dur=%s\n", queryDur) - } else { - add("fail", "query.blocked", fmt.Sprintf("dur>=%s", *timeout)) - fmt.Fprintf(out, "self-query fail query.blocked dur=%s\n", queryDur) - } + add("fail", "query.blocked", fmt.Sprintf("dur>=%s", *timeout)) + fmt.Fprintf(out, "self-query fail query.blocked dur=%s\n", queryDur) case qerr != nil: add("fail", "query.error", qerr.Error()) fmt.Fprintf(out, "self-query fail query.error err=%v\n", qerr) @@ -180,7 +139,6 @@ func RunDoctor(args []string, version string) int { fmt.Fprintln(out, "\n== Environment ==") fmt.Fprintf(out, "version: %s\n", version) - fmt.Fprintf(out, "index: %s (source=%s)\n", indexPath, indexSource) count, cerr := idx.DocCount() if cerr != nil { add("warn", "index.count", cerr.Error()) @@ -189,15 +147,6 @@ func RunDoctor(args []string, version string) int { fmt.Fprintf(out, "doc count: %d\n", count) } - storeDirs := storeDirsFor(resolver) - fresh := freshIndex(indexPath, storeDirs) - if fresh { - fmt.Fprintln(out, "index freshness: fresh") - } else { - add("fail", "index.stale", "store files are newer than the index") - fmt.Fprintln(out, "index freshness: STALE (store files newer than index)") - } - if rec, ok := install.LoadRecord(); ok { fmt.Fprintf(out, "install record: %s\n", recordSummary(rec)) } else { @@ -239,75 +188,6 @@ func printFindings(out io.Writer, findings []Finding) { } } -// storeDirsFor lists the knowledge dirs the resolver would index. -func storeDirsFor(res *projects.Resolver) []string { - var dirs []string - for _, ref := range res.Snapshot() { - switch ref.Kind { - case projects.KindOrg: - dirs = append(dirs, res.OrgKnowledgeDir(ref)) - default: - dirs = append(dirs, res.AgentsDir(ref)) - } - } - return dirs -} - -// freshIndex reports whether the index was written after the newest -// knowledge source file. Both sides are max-mtime over the respective -// trees; the store side counts source files only (*.yaml, *.md) so index -// artifacts inside the same tree never self-flag stale. -func freshIndex(indexPath string, storeDirs []string) bool { - indexM := treeMTime(indexPath) - storeM := time.Time{} - for _, d := range storeDirs { - if m := knowledgeMTime(d); m.After(storeM) { - storeM = m - } - } - if storeM.IsZero() { - return true - } - return !indexM.Before(storeM) -} - -func knowledgeMTime(root string) time.Time { - var newest time.Time - _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - if d.IsDir() { - return nil - } - if ext := filepath.Ext(d.Name()); ext != ".yaml" && ext != ".md" { - return nil - } - if info, ierr := d.Info(); ierr == nil && info.ModTime().After(newest) { - newest = info.ModTime() - } - return nil - }) - return newest -} - -func treeMTime(root string) time.Time { - var newest time.Time - _ = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return nil - } - if d.IsDir() { - return nil - } - if info, ierr := d.Info(); ierr == nil && info.ModTime().After(newest) { - newest = info.ModTime() - } - return nil - }) - return newest -} - func recordSummary(rec install.Record) string { parts := []string{"version=" + rec.Version} if len(rec.Roots) > 0 { diff --git a/internal/diag/doctor_test.go b/internal/diag/doctor_test.go index 277d210..6b872da 100644 --- a/internal/diag/doctor_test.go +++ b/internal/diag/doctor_test.go @@ -1,19 +1,12 @@ package diag import ( - "bytes" - "fmt" "io" - "log" "os" "path/filepath" - "runtime" "strings" - "sync" "testing" "time" - - "github.com/renderorange/knowledge-mcp/search" ) func TestWatchdogBlocked(t *testing.T) { @@ -40,13 +33,15 @@ func TestRunDoctorEmptyArgsFails(t *testing.T) { } } +const healthyYAML = "project: test\nversion: 1\nentries:\n - id: conv-001\n summary: test entry\n detail: body\n" + func TestRunDoctorHealthyFixture(t *testing.T) { proj := t.TempDir() agents := filepath.Join(proj, ".agents") if err := os.MkdirAll(agents, 0755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte("project: test\nversion: 1\nentries:\n - id: conv-001\n summary: test entry\n detail: body\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte(healthyYAML), 0644); err != nil { t.Fatal(err) } out := captureStdout(t, func() { @@ -55,17 +50,25 @@ func TestRunDoctorHealthyFixture(t *testing.T) { t.Errorf("RunDoctor() = %d, want 0", code) } }) - for _, want := range []string{"== Resolution ==", "== Startup trail ==", "== Environment ==", "findings:", "query.blocked"} { - if want == "query.blocked" { - if strings.Contains(out, want) { - t.Errorf("unexpected blocking finding: %s", out) - } - continue - } + for _, want := range []string{"== Resolution ==", "== Startup trail ==", "== Environment ==", "findings:"} { if !strings.Contains(out, want) { t.Errorf("missing %q in output:\n%s", want, out) } } + if strings.Contains(out, "query.blocked") { + t.Errorf("unexpected blocking finding: %s", out) + } + // The kick line must precede the self-query result: the self-query only + // returns once IndexAll closes the ready channel. + trail := strings.Index(out, "== Startup trail ==") + kick := strings.Index(out, "index.kick done") + query := strings.Index(out, "self-query") + if trail < 0 || kick < 0 || query < 0 { + t.Fatalf("missing trail/kick/self-query markers in output:\n%s", out) + } + if !(trail < kick && kick < query) { + t.Errorf("index.kick done out of order (trail=%d kick=%d query=%d):\n%s", trail, kick, query, out) + } } func TestRunDoctorUnresolvedPathFails(t *testing.T) { @@ -93,191 +96,53 @@ func TestRunDoctorSanitizesNothingBodies(t *testing.T) { } } -func TestRunDoctorEmptyEntriesStaysFresh(t *testing.T) { - name := "freshproj" - proj := filepath.Join(t.TempDir(), name) +func TestRunDoctorIndexFlagIgnored(t *testing.T) { + proj := t.TempDir() agents := filepath.Join(proj, ".agents") if err := os.MkdirAll(agents, 0755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte("project: test\nversion: 1\nentries: []\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte(healthyYAML), 0644); err != nil { t.Fatal(err) } + indexPath := filepath.Join(proj, "should-not-exist.index") out := captureStdout(t, func() { - code := RunDoctor([]string{"--project", proj, "--timeout", "10s"}, "test") + code := RunDoctor([]string{"--project", proj, "--index", indexPath, "--timeout", "10s"}, "test") if code != 0 { t.Errorf("RunDoctor() = %d, want 0", code) } }) - if !strings.Contains(out, "index freshness: fresh") { - t.Errorf("missing fresh line in output:\n%s", out) + if !strings.Contains(out, "--index is deprecated") { + t.Errorf("missing --index deprecation warning:\n%s", out) } - if strings.Contains(out, "index.stale") || strings.Contains(out, "STALE") { - t.Errorf("unexpected stale finding on empty store:\n%s", out) + if _, err := os.Stat(indexPath); !os.IsNotExist(err) { + t.Errorf("--index path was created despite being ignored: %v", err) } } -func TestRunDoctorWarmIndex(t *testing.T) { - name := "warmproj" - proj := filepath.Join(t.TempDir(), name) - agents := filepath.Join(proj, ".agents") - if err := os.MkdirAll(agents, 0755); err != nil { - t.Fatal(err) - } - // 60 entries across three files (30/20/10): big enough that the - // background IndexAll is still adding when RunDoctor finishes its - // report and hits teardown, so a Close/IndexAll race surfaces as - // stderr warnings instead of winning the scheduling race. A single - // entry is indexed before teardown and proves nothing. - for _, f := range []struct { - cat, prefix string - n int - }{ - {"conventions", "conv", 30}, - {"decisions", "dec", 20}, - {"subsystems", "sub", 10}, - } { - path := filepath.Join(agents, f.cat+".yaml") - if err := os.WriteFile(path, []byte(knowledgeYAML(f.prefix, f.n)), 0644); err != nil { - t.Fatal(err) - } - } - idx, err := search.NewIndex(filepath.Join(agents, ".index"), []string{name}) - if err != nil { - t.Fatalf("pre-populate warm index: %v", err) - } - if err := idx.Add(name+"/conv-001", search.SearchDocument{Summary: "test entry", Detail: "body", Category: "conventions", Project: name}); err != nil { - t.Fatalf("pre-populate add: %v", err) - } - if err := idx.Close(); err != nil { - t.Fatalf("pre-populate close: %v", err) - } - var logBuf syncBuffer - log.SetOutput(&logBuf) - defer log.SetOutput(os.Stderr) - out := captureStdout(t, func() { - code := RunDoctor([]string{"--project", proj, "--timeout", "10s"}, "test") - if code != 0 { - t.Errorf("RunDoctor() = %d, want 0", code) - } - }) - // Drain the leaked IndexAll writer before the test returns (bounded): - // the warm-path leak keeps it in flight past RunDoctor, and if - // t.TempDir() cleanup deletes its files under it, bleve emits - // persist-err warnings and can panic (zapx nil-map on the failed - // segment), taking the whole test binary down. A silent writer has - // no completion signal in the log, so poll its goroutine frame. - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) && indexAllRunning() { - time.Sleep(10 * time.Millisecond) - } - // Then let the leaked index's background persistence quiesce: scorch - // persister/merger goroutines outlive IndexAll, and a late .zap write - // racing t.TempDir() RemoveAll fails cleanup with "directory not - // empty". 200ms of index-tree quiet, bounded at 2s. - indexPath := filepath.Join(agents, ".index") - qDeadline := time.Now().Add(2 * time.Second) - prev := treeMTime(indexPath) - quietSince := time.Now() - for time.Now().Before(qDeadline) { - time.Sleep(50 * time.Millisecond) - cur := treeMTime(indexPath) - if cur.After(prev) { - prev = cur - quietSince = time.Now() - continue - } - if time.Since(quietSince) >= 200*time.Millisecond { - break - } - } - stderr := logBuf.String() - var tolerated []string - failed := false - if strings.Contains(stderr, "index is closed") { - t.Errorf("stderr contains %q: index closed while IndexAll in flight", "index is closed") - failed = true - } - for _, line := range strings.Split(stderr, "\n") { - if !strings.Contains(line, "failed to index") { - continue - } - if strings.Contains(line, "persist err:") { - // The one tolerated class: leaked writer racing t.TempDir() - // cleanup (this or a prior run's leaked goroutine). Never - // tolerate index is closed or a non-persist failure. - tolerated = append(tolerated, line) - continue - } - t.Errorf("unexpected index failure on stderr: %s", line) - failed = true - } - if failed || len(tolerated) > 0 { - t.Logf("captured stderr (%d tolerated persist-err lines):\n%s", len(tolerated), stderr) - } - trail := strings.Index(out, "== Startup trail ==") - kick := strings.Index(out, "index.kick done") - query := strings.Index(out, "self-query") - env := strings.Index(out, "== Environment ==") - findings := strings.Index(out, "findings:") - if trail < 0 || kick < 0 || query < 0 || env < 0 || findings < 0 { - t.Fatalf("missing markers in output:\n%s", out) - } - if !(trail < kick && kick < query && kick < env && kick < findings) { - t.Errorf("index.kick done out of order (trail=%d kick=%d query=%d env=%d findings=%d):\n%s", trail, kick, query, env, findings, out) - } - if strings.Contains(out, "index.stale") || strings.Contains(out, "STALE") { - t.Errorf("unexpected stale finding on warm store:\n%s", out) - } -} - -func TestRunDoctorIndexLocked(t *testing.T) { - name := "lockproj" - proj := filepath.Join(t.TempDir(), name) - agents := filepath.Join(proj, ".agents") - if err := os.MkdirAll(agents, 0755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte("project: test\nversion: 1\nentries:\n - id: conv-001\n summary: test entry\n detail: body\n"), 0644); err != nil { - t.Fatal(err) - } - holder, err := search.NewIndex(filepath.Join(agents, ".index"), []string{name}) - if err != nil { - t.Fatalf("hold index lock: %v", err) - } - t.Cleanup(func() { holder.Close() }) - out := captureStdout(t, func() { - code := RunDoctor([]string{"--project", proj, "--timeout", "500ms"}, "test") - if code != 1 { - t.Errorf("RunDoctor() = %d, want 1", code) - } - }) - if !strings.Contains(out, "fail") || !strings.Contains(out, "index.locked") { - t.Errorf("missing fail index.locked finding:\n%s", out) - } -} - -func TestRunDoctorNoIndexOnStartupWarnsQueryBlocked(t *testing.T) { - name := "warnproj" - proj := filepath.Join(t.TempDir(), name) +func TestRunDoctorNoIndexOnStartupIgnored(t *testing.T) { + proj := t.TempDir() agents := filepath.Join(proj, ".agents") if err := os.MkdirAll(agents, 0755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte("project: test\nversion: 1\nentries: []\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(agents, "conventions.yaml"), []byte(healthyYAML), 0644); err != nil { t.Fatal(err) } out := captureStdout(t, func() { - code := RunDoctor([]string{"--project", proj, "--no-index-on-startup", "--timeout", "300ms"}, "test") + code := RunDoctor([]string{"--project", proj, "--no-index-on-startup", "--timeout", "10s"}, "test") if code != 0 { t.Errorf("RunDoctor() = %d, want 0", code) } }) - for _, want := range []string{"warn query.blocked reason=no-index-on-startup", "findings:"} { + for _, want := range []string{"--no-index-on-startup is deprecated", "index.kick done"} { if !strings.Contains(out, want) { t.Errorf("missing %q in output:\n%s", want, out) } } + if strings.Contains(out, "query.blocked") { + t.Errorf("unexpected blocked finding:\n%s", out) + } } // captureStdout runs f while capturing os.Stdout (RunDoctor writes the @@ -299,46 +164,3 @@ func captureStdout(t *testing.T, f func()) string { } return string(data) } - -// knowledgeYAML renders n wrapper-form knowledge entries under the given -// id prefix (a bare YAML list loads as zero entries). -func knowledgeYAML(prefix string, n int) string { - var b strings.Builder - b.WriteString("project: test\nversion: 1\nentries:\n") - for i := 1; i <= n; i++ { - fmt.Fprintf(&b, " - id: %s-%03d\n summary: warm entry %s-%03d\n detail: body %d\n", prefix, i, prefix, i, i) - } - return b.String() -} - -// syncBuffer is a goroutine-safe log sink: the leaked IndexAll writer may -// log while the test polls and reads the captured stderr. -type syncBuffer struct { - mu sync.Mutex - buf bytes.Buffer -} - -func (b *syncBuffer) Write(p []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.Write(p) -} - -func (b *syncBuffer) String() string { - b.mu.Lock() - defer b.mu.Unlock() - return b.buf.String() -} - -// indexAllRunning reports whether any goroutine is still inside -// search.(*Index).IndexAll — the leaked warm-path writer. -func indexAllRunning() bool { - buf := make([]byte, 1<<20) - for { - n := runtime.Stack(buf, true) - if n < len(buf) { - return strings.Contains(string(buf[:n]), "search.(*Index).IndexAll") - } - buf = make([]byte, 2*len(buf)) - } -} From 6606ce0d31b927193e4cd4188d3aa2e49bf53e22 Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 11:27:19 -0400 Subject: [PATCH 5/6] GH-34: document in-memory search index --- README.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2974b55..5b068f2 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,6 @@ Layout: ``` / - .index/ # bleve index (default; --index wins) /.agents/ # per-project store (qualified names nest: # / -> ///.agents) /.agents/knowledge/ # org-level markdown @@ -147,8 +146,10 @@ Layout: - `init_knowledge` initializes the central store for resolvable paths and errors for others (add them via `--project`/`--root` first). - The store directory itself must exist and is never created for you. -- The store is single-writer: its search index and files do not support - concurrent access, so point only one server at a store. +- The store files are single-writer: their YAML reads/writes do not support + concurrent access across server instances, so point only one server at a + store for writing. Queries are safe from any instance — the search index + is per-process in-memory and never shared. ### Shared global store @@ -177,21 +178,25 @@ General notes: - Each `--root` discovers its immediate children as projects (one level deep — pass deeper directories as additional flags). - Duplicate project basenames across roots are addressed as `/` (e.g. `work/api`); `list_projects` shows which names need qualification. - `query_knowledge` accepts org root names to search that root's `.agents/knowledge/` files. Org-level documents are indexed **per `##` section**, so a query returns the matching section(s), not the entire file. -- Multi-entry configurations store the search index under `$XDG_STATE_HOME/knowledge-mcp/` (default `~/.local/state/knowledge-mcp/`). Single-flag configurations keep the index inside their own `.agents/` — unless `--store` is set, in which case the index lives at `/.index`. -- `--index ` overrides the index location in all modes. +- The search index is built **in memory** per server process at startup and + does not survive a restart. There is no index on disk anywhere, no lock, + and no contention — any number of instances may query concurrently. +- `--index ` is deprecated and ignored (the search index is in-memory). +- Old leftover `.agents/.index/` directories from earlier releases are inert + and can be deleted. ### Startup Options -`--no-index-on-startup` skips background indexing on startup. The server starts -immediately with stale or empty index data; queries block until indexing completes -if no stale data is available. Useful for large knowledge stores where startup -indexing would delay server readiness. +Both `--no-index-on-startup` and `--index` are deprecated no-ops. The server +always builds its search index from the store files at startup (backgrounded; +queries wait for it once). They remain accepted so existing configurations +keep working without changes. ### Debugging `knowledge-mcp debug` runs a one-shot startup diagnosis against the same flags as server mode and exits non-zero when it finds a problem (unresolved -project, stale or missing index, blocked queries): +project, blocked queries): ```bash knowledge-mcp debug --root /path/to/org @@ -199,9 +204,8 @@ knowledge-mcp debug --root /path/to/org The report has three sections — Resolution (projects/stores resolved), Startup trail (phase timings and a watchdog-bounded index self-query that -flags the cold-index query block), and Environment (version, index -location/source, doc count, freshness, install record) — followed by -`ok`/`warn`/`fail` findings. +flags a slow index kick), and Environment (version, doc count, install +record) — followed by `ok`/`warn`/`fail` findings. For live diagnostics, run the server with `--debug` (optionally `--log-file ` to tee). Lines go to stderr, which MCP clients such as From f905112185319024f5cf23459924266ca6d4417d Mon Sep 17 00:00:00 2001 From: Blaine Motsinger Date: Wed, 7 Oct 2026 12:00:06 -0400 Subject: [PATCH 6/6] GH-34: refresh self-store entries and polish review minors --- .agents/conventions.yaml | 4 ++-- .agents/decisions.yaml | 27 +++++++++++----------- .agents/subsystems.yaml | 44 ++++++++++++++++++------------------ README.md | 3 +-- internal/diag/doctor.go | 6 ++--- internal/diag/doctor_test.go | 12 ++++++---- projects/resolver.go | 2 +- 7 files changed, 49 insertions(+), 49 deletions(-) diff --git a/.agents/conventions.yaml b/.agents/conventions.yaml index fbdb1b0..b0c1871 100644 --- a/.agents/conventions.yaml +++ b/.agents/conventions.yaml @@ -2,7 +2,7 @@ project: knowledge-mcp version: 1 entries: - id: conv-001 - summary: Go project standards: gofmt, go vet, go test, GH-N commits + summary: "Go project standards: gofmt, go vet, go test, GH-N commits" detail: |- Format: gofmt (make fmt runs gofmt -s -w .). gofmt -l must output nothing. Verify: go test ./... and go vet ./... must pass before work is claimed done; go build ./... must compile all packages. @@ -13,7 +13,7 @@ entries: source: code review, 2026-09-08 date: "2026-09-08" - id: conv-002 - summary: Knowledge entry content rules for this project's own store + summary: "Knowledge entry content rules for this project's own store" detail: |- Detail is 10-20 lines of query-able facts, not source copies. Summary is a topic line (max 100 chars); enforcement text goes in the rule field, which list_knowledge surfaces as constraints. diff --git a/.agents/decisions.yaml b/.agents/decisions.yaml index 46e9621..3786242 100644 --- a/.agents/decisions.yaml +++ b/.agents/decisions.yaml @@ -2,16 +2,16 @@ project: knowledge-mcp version: 1 entries: - id: dec-001 - summary: Org-level knowledge is indexed per markdown section, not as whole files + summary: "Org-level knowledge is indexed per markdown section, not as whole files" detail: |- Whole-file indexing made org queries return 5-6K tokens of unrelated content (measured 2026-09-08: "daisy test-stability" on chaon-io returned full architecture.md and review.md). Split .agents/knowledge/*.md on "## " headings; each section is its own document with key org-::. list_knowledge for an org groups sections by source file; --global entries are merged into org listings with a (global) label. - Required metaFileVersion bump to force index rebuilds on existing installs. - source: user-requested token-savings fix, 2026-09-08 - date: "2026-09-08" + No version bump is needed anymore: the index is rebuilt from the store files on every startup (GH-34 in-memory index). + source: user-requested token-savings fix, 2026-09-08; GH-34 update, 2026-10-07 + date: "2026-10-07" - id: dec-002 - summary: Merged global entries carry explicit (global) provenance in list output + summary: "Merged global entries carry explicit (global) provenance in list output" detail: |- list_knowledge for any non-global project prepends _global entries; previously they were indistinguishable from the project's own entries. Section headers now carry a " (global)" suffix; projects with no local entries get an explicit "no project-specific knowledge entries found" note. @@ -19,7 +19,7 @@ entries: source: user-requested provenance fix, 2026-09-08 date: "2026-09-08" - id: dec-003 - summary: Query text is escaped so special characters are always literal + summary: "Query text is escaped so special characters are always literal" detail: |- User queries are plain text, not lucene syntax; characters like : ( ) && || " were previously interpreted as operators. escapeQueryString escapes syntax-active characters (context-aware for +/- and &&/||); mid-word hyphens like test-stability stay intact. @@ -27,12 +27,11 @@ entries: source: user-requested robustness fix, 2026-09-08 date: "2026-09-08" - id: dec-004 - summary: Background indexing with --no-index-on-startup for large stores + summary: "Startup indexing: always built in memory (no skip flag)" detail: |- - Synchronous indexing on startup blocked server readiness for large knowledge stores. - Changed to background goroutine (go idx.IndexAll(resolver)) with ready channel for query synchronization. - Added --no-index-on-startup flag to skip indexing entirely; useful when startup delay is unacceptable. - Queries block on ready channel until indexing completes; stale data remains available if index has data. - readyOnce (sync.Once) prevents double-close panic when NewIndex already closed ready channel. - source: GH-25 background indexing, 2026-09-20 - date: "2026-09-20" \ No newline at end of file + The former background-indexing flag --no-index-on-startup is now a deprecated no-op (accepted, warns, ignored). + The index is always built in memory at startup; queries block on the ready channel until IndexAll completes. + readyOnce (sync.Once) prevents double-close panics. + Recorded so a skip-indexing flag is not reintroduced. + source: GH-34 memory-only index, 2026-10-07 + date: "2026-10-07" \ No newline at end of file diff --git a/.agents/subsystems.yaml b/.agents/subsystems.yaml index a259917..3bce612 100644 --- a/.agents/subsystems.yaml +++ b/.agents/subsystems.yaml @@ -2,28 +2,28 @@ project: knowledge-mcp version: 1 entries: - id: sub-001 - summary: Bleve search index: lifecycle, rebuild triggers, org section docs + summary: "Search index: per-process in-memory bleve, scoped keys, org sections" detail: |- - Located at -derived path: single-flag configs use /.agents/.index; multi-entry configs use $XDG_STATE_HOME/knowledge-mcp/.index; --index overrides. - metaFileVersion (search/index.go, currently 3) + recorded project-name set are written to .meta.json; changed version or names force a wipe+rebuild from yaml source of truth. - Corruption auto-recovery: bleve.Open failure deletes index dir and rebuilds (search/index.go NewIndex). - Documents: summary, detail, rule, category, project fields; scoped keys /. Query results de-scope the ID. - Org-level docs (.agents/knowledge/*.md) are split on "## " headings into sections with keys org-::; content before first heading becomes Overview (main.go splitSections). - User query text is escaped (search/index.go escapeQueryString) so special characters are literal, never lucene syntax. - source: code review, 2026-09-08 - date: "2026-09-08" + The index is built in memory (bleve.NewMemOnly) at server startup from the store YAML/MD files — every restart re-indexes. + There is no disk state, no index lock, and any number of instances may query concurrently. + Documents carry summary, detail, rule, category, project with scoped keys /. + Query results de-scope the ID. + Org-level docs (.agents/knowledge/*.md) are split on "## " headings with keys org-::. + User query text is escaped (escapeQueryString) so special characters are literal, never lucene syntax. + --index and --no-index-on-startup are accepted warning-logged no-ops. + source: GH-34 memory-only index, 2026-10-07 + date: "2026-10-07" - id: sub-002 - summary: Project resolver: kinds, addressing, dynamic fallback + summary: "Project resolver: kinds, addressing, dynamic fallback" detail: |- projects/resolver.go builds refs: KindProject (has .agents/ yaml), KindOrg (root with .agents/knowledge/), KindGlobal (--global path, addressed _global). Roots scan immediate children only, skipping .agents and .git; symlinks followed; unreadable entries warn. Addressing: bare name when unique, else /; org roots always win their bare name; duplicate root basenames are a startup error. Resolve falls back dynamically for paths created after startup (resolveDynamicBare/Qualified); Covers checks known paths and root prefixes. - Index location is keyed by hash of canonical root/project paths (github sha256) so different configs get different indexes. - source: code review, 2026-09-08 - date: "2026-09-08" + source: code review, 2026-09-08; GH-34 update, 2026-10-07 + date: "2026-10-07" - id: sub-003 - summary: Knowledge store: atomic writes, per-file locks, entry lifecycle + summary: "Knowledge store: atomic writes, per-file locks, entry lifecycle" detail: |- knowledge/store.go: Save writes temp file, fsyncs, renames (atomic); LoadOrCreate returns empty KnowledgeFile when missing. tools package keeps per-path RWMutexes (fileLocks) around read-modify-write cycles; write/update lock, list takes RLock. @@ -34,12 +34,12 @@ entries: source: code review, 2026-09-08 date: "2026-09-08" - id: sub-004 - summary: Background indexing with --no-index-on-startup flag + summary: "Startup indexing: always-on background kick, no skip flag" detail: |- - IndexAll runs as a goroutine (go idx.IndexAll(resolver)) when --no-index-on-startup is false (default). - When --no-index-on-startup is true, indexing is skipped entirely; server starts with stale or empty index. - Queries block on ready channel until indexing completes; ready channel closes when IndexAll finishes or if index has stale data. - readyOnce (sync.Once) prevents double-close panic if NewIndex already closed ready channel. - Useful for large knowledge stores where startup indexing would delay server readiness. - source: GH-25 background indexing, 2026-09-20 - date: "2026-09-20" \ No newline at end of file + IndexAll always runs in a background goroutine at startup (there is no skip flag). + Queries block on the ready channel until IndexAll completes. + readyOnce (sync.Once) prevents a double-close panic on the ready channel. + --no-index-on-startup and --index are deprecated, accepted, warn, and do nothing. + The index lives only as long as the process and is rebuilt from the store files on every startup. + source: GH-34 memory-only index, 2026-10-07 + date: "2026-10-07" \ No newline at end of file diff --git a/README.md b/README.md index 5b068f2..742bd72 100644 --- a/README.md +++ b/README.md @@ -182,8 +182,7 @@ General notes: does not survive a restart. There is no index on disk anywhere, no lock, and no contention — any number of instances may query concurrently. - `--index ` is deprecated and ignored (the search index is in-memory). -- Old leftover `.agents/.index/` directories from earlier releases are inert - and can be deleted. +- Old leftover `.agents/.index/` and `/.index/` directories and `$XDG_STATE_HOME/knowledge-mcp/` artifacts from earlier releases are inert and can be deleted. ### Startup Options diff --git a/internal/diag/doctor.go b/internal/diag/doctor.go index e1d9bd0..8944885 100644 --- a/internal/diag/doctor.go +++ b/internal/diag/doctor.go @@ -47,7 +47,7 @@ func RunDoctor(args []string, version string) int { globalPath := fs.String("global", "", "Path to a global knowledge store shared across all projects") indexOverride := fs.String("index", "", "Deprecated: the search index is in-memory; accepted and ignored") storeDir := fs.String("store", "", "Central directory for all knowledge stores") - noIndexOnStartup := fs.Bool("no-index-on-startup", false, "Deprecated: the index is always built; accepted and ignored") + noIndexOnStartup := fs.Bool("no-index-on-startup", false, "Deprecated: the index is always built at startup; accepted and ignored") timeout := fs.Duration("timeout", 5*time.Second, "Self-query watchdog deadline") logFile := fs.String("log-file", "", "Tee the report to this file") fs.Var(&roots, "root", "Org root whose immediate children are projects (repeatable)") @@ -103,7 +103,7 @@ func RunDoctor(args []string, version string) int { fmt.Fprintln(out, "warning: --index is deprecated and ignored; the search index is per-process in-memory") } if *noIndexOnStartup { - fmt.Fprintln(out, "warning: --no-index-on-startup is deprecated and ignored; the index is always built") + fmt.Fprintln(out, "warning: --no-index-on-startup is deprecated and ignored; the index is always built at startup") } tIndex := time.Now() @@ -117,7 +117,7 @@ func RunDoctor(args []string, version string) int { fmt.Fprintln(out, "\n== Startup trail ==") fmt.Fprintf(out, "resolve.done dur=%s\n", resolveDur) - fmt.Fprintf(out, "index.kick done (in-memory, dur=%s)\n", time.Since(tIndex).Round(time.Millisecond)) + fmt.Fprintf(out, "index.kick started (in-memory, dur=%s)\n", time.Since(tIndex).Round(time.Millisecond)) go idx.IndexAll(resolver) tQuery := time.Now() diff --git a/internal/diag/doctor_test.go b/internal/diag/doctor_test.go index 6b872da..5ad1376 100644 --- a/internal/diag/doctor_test.go +++ b/internal/diag/doctor_test.go @@ -58,16 +58,18 @@ func TestRunDoctorHealthyFixture(t *testing.T) { if strings.Contains(out, "query.blocked") { t.Errorf("unexpected blocking finding: %s", out) } - // The kick line must precede the self-query result: the self-query only - // returns once IndexAll closes the ready channel. + // The ordering assertion pins synchronous print order: the kick line is + // printed before the self-query is issued. The ready-channel gate itself + // is proven by the absence of a query.blocked finding on the real ready + // channel (Query blocks on <-i.ready). trail := strings.Index(out, "== Startup trail ==") - kick := strings.Index(out, "index.kick done") + kick := strings.Index(out, "index.kick started") query := strings.Index(out, "self-query") if trail < 0 || kick < 0 || query < 0 { t.Fatalf("missing trail/kick/self-query markers in output:\n%s", out) } if !(trail < kick && kick < query) { - t.Errorf("index.kick done out of order (trail=%d kick=%d query=%d):\n%s", trail, kick, query, out) + t.Errorf("index.kick started out of order (trail=%d kick=%d query=%d):\n%s", trail, kick, query, out) } } @@ -135,7 +137,7 @@ func TestRunDoctorNoIndexOnStartupIgnored(t *testing.T) { t.Errorf("RunDoctor() = %d, want 0", code) } }) - for _, want := range []string{"--no-index-on-startup is deprecated", "index.kick done"} { + for _, want := range []string{"--no-index-on-startup is deprecated", "index.kick started"} { if !strings.Contains(out, want) { t.Errorf("missing %q in output:\n%s", want, out) } diff --git a/projects/resolver.go b/projects/resolver.go index 964b1d3..c676504 100644 --- a/projects/resolver.go +++ b/projects/resolver.go @@ -134,7 +134,7 @@ func BuildWithStore(roots, projects []string, global, store string) (*Resolver, } if storePath != "" && child.Name == ".index" { warnings = append(warnings, fmt.Sprintf( - "skipping %s under root %s: name collides with the --store index directory", + "skipping %s under root %s: name is a reserved knowledge-mcp artifact directory (legacy store index)", child.Name, rp)) continue }