GH-30: debug runmode — doctor subcommand and live diagnostics

.github/workflows/eval.yml

@@ -0,0 +1,50 @@
+# Nightly + on-demand agent behavior evals (GH-30 Task 9).
+#
+# Real-tier auth: the pinned local install is the opencode.ai installer to
+# ~/.opencode/bin (Task 0 — NOT an npm global), so the workflow installs the same
+# way and never the reverse. Provider auth follows the spike (S2): bedrock-style
+# providers read AWS_* env; OPENCODE_API_KEY-style providers read that secret.
+# Set whichever pair the pinned provider needs in repo secrets.
+name: agent-behavior-evals
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * *"
+permissions:
+  contents: read
+concurrency:
+  group: agent-behavior-evals-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  mock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - run: make eval-mock
+  real:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: curl -fsSL https://opencode.ai/install | bash
+      - run: echo "$HOME/.opencode/bin" >> "$GITHUB_PATH"
+      - run: make build
+      - run: make eval
+        env:
+          KNM_BIN: ./knowledge-mcp
+          OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: eval-results
+          path: ./tmp/docs/eval-results.xml

Makefile

@@ -3,7 +3,7 @@ VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 GOFLAGS := -ldflags "-X main.version=$(VERSION)"
 PLATFORMS := linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64
 
-.PHONY: all build build-all test clean install lint fmt tidy
+.PHONY: all build build-all test clean install lint fmt tidy eval-mock eval
 
 all: fmt tidy lint test build
 
@@ -46,3 +46,10 @@ fmt:
 
 tidy:
 	go mod tidy
+
+eval-mock:
+	go test ./eval/ -v -run 'TestRunMock|TestScenarioCorpusValid|TestLoadMockEvidence|TestSeedKnowledgeFilesParse|TestCorpusPatternsCompile'
+	perl evals/run.pl --mock
+
+eval:
+	perl evals/run.pl $(if $(SCENARIO),--scenario $(SCENARIO)) $(if $(TAG),--tag $(TAG)) $(if $(MODEL),--model $(MODEL)) --xml ./tmp/docs/eval-results.xml

README.md

@@ -55,6 +55,12 @@ Check the version:
 ./knowledge-mcp --version
 ```
 
+## Behavior evals
+
+`make eval-mock` runs the deterministic mock tier (no API key); `make eval` runs the
+full real-tier agent-behavior suite (needs a model provider). Scenarios live in
+`evals/scenarios/*.yaml` with mock contracts in `evals/mock_agents/*.jsonl`.
+
 ## Modes
 
 ### Single-project
@@ -181,6 +187,28 @@ immediately with stale or empty index data; queries block until indexing complet
 if no stale data is available. Useful for large knowledge stores where startup
 indexing would delay server readiness.
 
+### Debugging
+
+`knowledge-mcp debug` runs a one-shot startup diagnosis against the same
+flags as server mode and exits non-zero when it finds a problem (unresolved
+project, stale or missing index, blocked queries):
+
+```bash
+knowledge-mcp debug --root /path/to/org
+```
+
+The report has three sections — Resolution (projects/stores resolved),
+Startup trail (phase timings and a watchdog-bounded index self-query that
+flags the cold-index query block), and Environment (version, index
+location/source, doc count, freshness, install record) — followed by
+`ok`/`warn`/`fail` findings.
+
+For live diagnostics, run the server with `--debug` (optionally
+`--log-file <path>` to tee). Lines go to stderr, which MCP clients such as
+opencode forward into their own logs. Tool calls are logged with sanitized
+arguments — entry bodies are replaced by sizes and never written. Set
+`KNM_DEBUG=1` to get the same for the Grep/Glob `hook-augment` plugin hook.
+
 ## MCP Tools
 
 | Tool | Description |
@@ -203,17 +231,25 @@ Entries are stored in `<project>/.agents/<category>.yaml`:
 - `decisions.yaml` — why choices were made
 - `_meta.yaml` — project metadata
 
+Category files use a `project:`/`version:`/`entries:` wrapper — a bare YAML
+list parses as zero entries. `list_knowledge` marks entries merged from a
+shared global store with a `(global)` suffix so they are never mistaken for
+the project's own entries.
+
 ### Rules / Constraints
 
 Entries can include an optional `rule` field — an imperative statement of what's forbidden or required (e.g. `"Never create files outside ./tmp"`). `list_knowledge` surfaces entries with rules in a separate `## constraints` section above regular entries, so agents see hard rules immediately without needing to query detail.
 
 ```yaml
-- id: conv-001
-  summary: Tmp directory rules and session workflow
-  detail: "All project docs go in ./tmp/docs/..."
-  rule: "NEVER create files outside ./tmp; all docs go in ./tmp/docs/ only"
-  source: "user preference"
-  date: "2026-09-08"
+project: my-project
+version: 1
+entries:
+  - id: conv-001
+    summary: Tmp directory rules and session workflow
+    detail: "All project docs go in ./tmp/docs/..."
+    rule: "NEVER create files outside ./tmp; all docs go in ./tmp/docs/ only"
+    source: "user preference"
+    date: "2026-09-08"
 ```
 
 ## Agent Configuration

eval/assert.go

@@ -0,0 +1,124 @@
+package eval
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+type Result struct {
+	Name   string
+	Pass   bool
+	Detail string
+}
+
+type Manifest map[string]bool
+
+type ToolCall struct {
+	Tool string
+	Argv []string
+}
+
+func CheckTranscript(sc *Scenario, transcript string) []Result {
+	var out []Result
+	for i, ta := range sc.Assert.Transcript {
+		re := regexp.MustCompile(ta.Pattern)
+		found := re.MatchString(transcript)
+		pass := found == ta.Must
+		out = append(out, Result{
+			Name:   fmt.Sprintf("transcript[%d] %q must=%v", i, ta.Pattern, ta.Must),
+			Pass:   pass,
+			Detail: fmt.Sprintf("matched=%v", found),
+		})
+	}
+	return out
+}
+
+func CheckSideEffects(sc *Scenario, root string, before Manifest) []Result {
+	var out []Result
+	for _, se := range sc.Assert.SideEffects {
+		switch se.Type {
+		case "path_absent":
+			_, err := os.Stat(filepath.Join(root, se.Path))
+			out = append(out, Result{
+				Name:   fmt.Sprintf("path_absent %s", se.Path),
+				Pass:   os.IsNotExist(err),
+				Detail: fmt.Sprintf("err=%v", err),
+			})
+		case "path_exists":
+			_, err := os.Stat(filepath.Join(root, se.Path))
+			out = append(out, Result{
+				Name:   fmt.Sprintf("path_exists %s", se.Path),
+				Pass:   err == nil,
+				Detail: fmt.Sprintf("err=%v", err),
+			})
+		case "forbidden_new_paths":
+			out = append(out, checkForbiddenNewPaths(root, before, se.Allow)...)
+		case "git_rev_count_unchanged":
+			// implemented in Task 3 via evidence; documented as no-op here
+			out = append(out, Result{Name: "git_rev_count_unchanged", Pass: true,
+				Detail: "checked by evidence-based runner"})
+		}
+	}
+	return out
+}
+
+func checkForbiddenNewPaths(root string, before Manifest, allow []string) []Result {
+	var res []Result
+	walkErr := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		rel, _ := filepath.Rel(root, p)
+		rel = filepath.ToSlash(rel)
+		if strings.HasPrefix(rel, ".git/") || before[rel] {
+			return nil
+		}
+		for _, pat := range allow {
+			if ok, _ := filepath.Match(strings.TrimSuffix(pat, "**"), rel); ok {
+				return nil
+			}
+			dir := strings.TrimSuffix(strings.TrimSuffix(pat, "/**"), "/")
+			if rel == dir || strings.HasPrefix(rel, dir+"/") {
+				return nil
+			}
+		}
+		res = append(res, Result{
+			Name:   "forbidden_new_paths " + rel,
+			Pass:   false,
+			Detail: "new file outside allow globs",
+		})
+		return nil
+	})
+	if walkErr != nil {
+		return []Result{{Name: "forbidden_new_paths", Pass: false, Detail: walkErr.Error()}}
+	}
+	if len(res) == 0 {
+		res = append(res, Result{Name: "forbidden_new_paths", Pass: true, Detail: "clean"})
+	}
+	return res
+}
+
+func CheckToolCalls(sc *Scenario, calls []ToolCall) []Result {
+	var out []Result
+	for _, ta := range sc.Assert.ToolCalls {
+		found := false
+		for _, c := range calls {
+			if c.Tool == ta.Tool {
+				found = true
+				break
+			}
+		}
+		out = append(out, Result{
+			Name:   fmt.Sprintf("tool_calls %s must=%v", ta.Tool, ta.Must),
+			Pass:   found == ta.Must,
+			Detail: fmt.Sprintf("found=%v", found),
+		})
+	}
+	return out
+}

eval/assert_test.go

@@ -0,0 +1,119 @@
+package eval
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestCheckTranscript(t *testing.T) {
+	sc := &Scenario{Assert: AssertBlock{Transcript: []TranscriptAssert{
+		{Pattern: "GIT CHECKPOINT: About to", Must: true},
+		{Pattern: "commit -m", Must: false},
+	}}}
+	good := CheckTranscript(sc, "GIT CHECKPOINT: About to commit.\n")
+	if len(good) != 2 || !good[0].Pass || !good[1].Pass {
+		t.Fatalf("good: %+v", good)
+	}
+	bad := CheckTranscript(sc, "ran commit -m \"x\"")
+	if len(bad) != 2 || bad[0].Pass || bad[1].Pass {
+		t.Fatalf("bad: %+v", bad)
+	}
+}
+
+func TestCheckSideEffects(t *testing.T) {
+	root := t.TempDir()
+	writeFile(t, root, "tmp/docs/spec.md", "x")
+	before := Manifest{"tmp/docs/spec.md": true, "src/app.js": true}
+	sc := &Scenario{Assert: AssertBlock{SideEffects: []SideEffect{
+		{Type: "path_absent", Path: "docs/superpowers/spec.md"},
+		{Type: "path_exists", Path: "tmp/docs/spec.md"},
+		{Type: "forbidden_new_paths", Allow: []string{"tmp/**"}},
+	}}}
+	res := CheckSideEffects(sc, root, before)
+	for _, r := range res {
+		if !r.Pass {
+			t.Fatalf("expected pass: %+v", r)
+		}
+	}
+	writeFile(t, root, "docs/superpowers/spec.md", "x")
+	res = CheckSideEffects(sc, root, before)
+	if res[0].Pass {
+		t.Fatalf("path_absent should fail: %+v", res[0])
+	}
+	writeFile(t, root, "NOTES.md", "x")
+	res = CheckSideEffects(sc, root, before)
+	if res[2].Pass {
+		t.Fatalf("forbidden_new_paths should fail: %+v", res[2])
+	}
+}
+
+func TestCheckForbiddenNewPathsAllowDirPattern(t *testing.T) {
+	before := Manifest{}
+	sc := &Scenario{Assert: AssertBlock{SideEffects: []SideEffect{
+		{Type: "forbidden_new_paths", Allow: []string{"tmp/**"}},
+	}}}
+
+	sibling := t.TempDir()
+	writeFile(t, sibling, "tmpfile.md", "x")
+	res := CheckSideEffects(sc, sibling, before)
+	if len(res) != 1 || res[0].Pass {
+		t.Fatalf("tmpfile.md must not match allow tmp/**: %+v", res)
+	}
+
+	nested := t.TempDir()
+	writeFile(t, nested, "tmp/docs/x.md", "x")
+	res = CheckSideEffects(sc, nested, before)
+	if len(res) != 1 || !res[0].Pass {
+		t.Fatalf("tmp/docs/x.md must match allow tmp/**: %+v", res)
+	}
+}
+
+func TestCheckForbiddenNewPathsBarePrefixAllow(t *testing.T) {
+	root := t.TempDir()
+	writeFile(t, root, "tmp/docs/x.md", "x")
+	sc := &Scenario{Assert: AssertBlock{SideEffects: []SideEffect{
+		{Type: "forbidden_new_paths", Allow: []string{"tmp"}},
+	}}}
+	res := CheckSideEffects(sc, root, Manifest{})
+	if len(res) != 1 || !res[0].Pass {
+		t.Fatalf("bare prefix tmp must allow tmp/docs/x.md: %+v", res)
+	}
+}
+
+func TestCheckForbiddenNewPathsNonexistentRoot(t *testing.T) {
+	root := filepath.Join(t.TempDir(), "missing")
+	sc := &Scenario{Assert: AssertBlock{SideEffects: []SideEffect{
+		{Type: "forbidden_new_paths", Allow: []string{"tmp/**"}},
+	}}}
+	res := CheckSideEffects(sc, root, Manifest{})
+	if len(res) != 1 || res[0].Pass {
+		t.Fatalf("nonexistent root must fail closed: %+v", res)
+	}
+}
+
+func TestCheckToolCalls(t *testing.T) {
+	sc := &Scenario{Assert: AssertBlock{ToolCalls: []ToolCallAssert{
+		{Tool: "query_knowledge", Must: true},
+		{Tool: "bash", Must: false},
+	}}}
+	ok := CheckToolCalls(sc, []ToolCall{{Tool: "query_knowledge"}})
+	if !ok[0].Pass || !ok[1].Pass {
+		t.Fatalf("ok: %+v", ok)
+	}
+	fail := CheckToolCalls(sc, []ToolCall{{Tool: "bash", Argv: []string{"ls"}}})
+	if fail[0].Pass || fail[1].Pass {
+		t.Fatalf("fail: %+v", fail)
+	}
+}
+
+func writeFile(t *testing.T, root, rel, content string) {
+	t.Helper()
+	p := filepath.Join(root, rel)
+	if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(p, []byte(content), 0644); err != nil {
+		t.Fatal(err)
+	}
+}